
    ^j                        d dl Z d dlmZ d dlZd dlmZ d dlmc mZ d dl	m
Z
 ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z? ddl@mAZA  e.j                  eC      ZDe
 G d de             ZEe
 G d de             ZFe
 G d de             ZGe
 G d  d!e             ZH G d" d#e8      ZI G d$ d%e5      ZJ G d& d'e<      ZK G d( d)ej                        ZM	 	 did*ej                  d+ej                  d,ej                  d-ej                  d.ej                  dz  d/eOd0eOd1ej                  dz  fd2ZP G d3 d4ej                        ZQ G d5 d6e7      ZR G d7 d8e?      ZS G d9 d:ej                        ZT G d; d<ej                        ZU G d= d>ej                        ZV ed?      	 	 djd@ej                  dAej                  dBej                  dCej                  dz  dDeXdz  f
dE       ZY edF      	 	 djd@ej                  dBej                  dCej                  dz  dDeXdz  fdG       ZZ eeYeZg       G dH dIej                               Z[ G dJ dKe!      Z\e, G dL dMe'             Z]e, G dN dOe]             Z^ G dP dQe6      Z_ G dR dSe:      Z` G dT dUe]      Za G dV dWej                        ZbdXecdYedec   fdZZe	 dkd[ecd\ecd]ecd^ecdYej                  f
d_Zg G d` dae]      Zh e,dbc       G dd dee]             Zi e,dbc       G df dge]e             Zjg dhZky)l    N)Callable)strict   )initialization)ACT2FN)CacheDynamicCache)PreTrainedConfig)GenerationMixin)use_kernel_func_from_hubuse_kernelized_func)force_accelerate_hooks)create_causal_maskcreate_recurrent_attention_mask!create_sliding_window_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleloggingtorch_compilable_check)merge_with_config_defaults)capture_outputs   )Gemma3CausalLMOutputWithPastGemma3ForCausalLM	Gemma3MLPGemma3ModelOutputWithPast)HiggsAudioV2Embeddings)LlamaRMSNorm	repeat_kv)MixtralExperts)apply_mask_to_padding_statesc                   ,    e Zd ZU dZdZddddddddddddZd	gd
gfddgdgfdgdgfdZdddddZddddddZdZ	e
ed<   dZe
dz  ed<   dZe
ed<   dZe
ed<   dZe
ed <   d!Ze
ed"<   d#Ze
ed$<   dZe
ed%<   d&Ze
ed'<   d#Ze
ed(<   d)Ze
ed<   d&Ze
ed*<   d+Ze
ed,<   dZe
dz  ed-<   d.Zeed/<   dZee
   dz  ed0<   dZee   dz  ed1<   d2Ze
ed<   d3Z eed4<   d5Z!e
ed<   dZ"ee   dz  ed6<   d7Z#e
ed8<   d9Z$eed:<   d;Z%e
ed<<   d=Z&e
ed<   d>Z'e
ed?<   d@Z(e
edA<   dBZ)e*edC<   dDZ+eedE<   dFZ,eed<   d3Z-eedG<   dHZ.eedI<   dJZ/eedK<   dZ0e
dz  edL<   dMZ1e
dz  edN<   d@Z2e
dz  edO<   dZ3e
dz  edP<   dQZ4e*edR<   dBZ5e*edS<   dZ6ee
   dz  edT<    fdUZ7e8dV        Z9e8dW        Z: xZ;S )XInklingTextConfiginkling_texttext_configembedding_rowwisecolwiserowwisemoe_tp_experts
all_reduce)embed_tokens!layers.*.mlp.experts.gate_up_projlayers.*.mlp.experts.down_projlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)r2   layersnorm	ep_routergrouped_gemm)zlayers.*.mlp.gater3   r4   r5   logits_mup_width_multipliersliding_window_sizen_routed_expertsconv_kernel_sizemax_position_embeddings)embedding_multipliersliding_windownum_local_expertssconv_kernel_sizemodel_max_lengthi@ 
vocab_sizeNunpadded_vocab_size   hidden_sizeB   num_hidden_layers@   num_attention_heads   num_key_value_heads   head_dimswa_num_attention_heads   swa_num_key_value_headsswa_head_dimi   d_rel   
rel_extentlog_scaling_n_floorg?log_scaling_alphalocal_layer_idslayer_typesi   ư>rms_norm_eps   mlp_layer_typesi `  intermediate_sizesilu
hidden_acti   moe_intermediate_size      num_experts_per_tokr   n_shared_expertsTshared_expert_sinkg       @route_scaleg      8@rms_norm_eps_moe_gate        attention_dropout{Gz?initializer_rangepad_token_id   bos_token_ideos_token_idnum_mtp_layersFchain_hidden_post_normmtp_hidden_states_firstmtp_local_layer_idsc                 F   | j                   {| j                  t        | j                        }n+t        | j                        D ch c]  }|dz   dz  s| }}t        | j                        D cg c]
  }||v rdnd c}| _         | j
                  A|j                  dd      }t        | j                        D cg c]  }||k  rdnd c}| _        |j                  d	      |j                  d	      | _        d
| _	        t        | ,  di | y c c}w c c}w c c}w )Nrs   rh   hybrid_slidinghybriddense_mlp_idxr   densesparsedense_intermediate_sizera    )r^   r]   setrangerM   rb   popgetrc   number_of_conv_statessuper__post_init__)selfkwargsr]   ir}   	__class__s        v/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/inkling/modular_inkling.pyr   zInklingTextConfig.__post_init__   s   ###/"%d&:&:";.3D4J4J.K"[PQTUPUYZ{1"["[PUVZVlVlPm KLA$8 hF D '"JJ:MX]^b^t^tXu#vSTq=/@Gh$N#vD ::/0<%+ZZ0I%JD" &'"'' #\ 
 $ws   DD2D>Dc                     | j                   O| j                  dg| j                   z  S t        | j                         D cg c]  }|| j                  v rdnd c}S y c c}w )Nr|   r{   )rv   ry   r   )r   r   s     r   mtp_layer_typesz!InklingTextConfig.mtp_layer_types   sr    *''/ zD$7$777 ^ccgcvcv]wXYT-E-E(E$8S  s    Ac                 <    | j                   dg| j                   z  S y )Nr~   )rv   )r   s    r   mtp_mlp_layer_typesz%InklingTextConfig.mtp_mlp_layer_types   s$    *9t2222    )<__name__
__module____qualname__
model_typebase_config_keybase_model_tp_planbase_model_pp_planbase_model_ep_planattribute_maprH   int__annotations__rI   rK   rM   rO   rQ   rS   rT   rV   rW   r?   rX   rZ   r[   r\   floatr]   listr^   strrB   r`   rA   rb   rc   re   rf   r@   ri   rj   rk   boolrl   r>   rm   ro   rq   rr   rt   ru   rv   rw   rx   ry   r   propertyr   r   __classcell__r   s   @r   r*   r*   <   s   J#O+-6*3 01:/81:'3"+ )"+ &(9:#%568IJ!"_$56 )-;*8 0	 !>// 05	M J&*t*Ks!!  Hc#%S%#%S%L#""E3OJ&*t*"u"(,OT#Y%,$(KcT!(#)S)L%c(,OT#Y%,"s"J!%3%c  c##K)--#'5'"u"#u##L#*# L#*  L#* !%NC$J%#(D($(T(,0cT)0(*    r   r*   c                   h    e Zd ZU dZdZddddZdZeed<   dZ	eed<   d	Z
eed<   d
Zeed<   dZeed<   y)InklingAudioConfiginkling_audioaudio_config
n_mel_binsmel_vocab_sizetext_hidden_size)num_codebookscodebook_sizerK   P   rg   rJ   r_   r`   rp   rq   N)r   r   r   r   r   r   r   r   r   r   r   r`   r   rq   r   r   r   r   r      sR     J$O%))M JNC c L%#u#r   r   c                       e Zd ZU dZdZddiZdZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)InklingVisionConfiginkling_visionvision_configrM   n_layersrJ   r   (   
patch_sizer   temporal_patch_sizer   num_channelsrY   rK      rU   rO   r_   r`   rp   rq   N)r   r   r   r   r   r   r   r   r   r   r   r   rK   rM   rO   r`   r   rq   r   r   r   r   r      su    !J%O(*5M c J  L#Ks!!L%#u#r   r   c                        e Zd ZU dZdZeeedZdZ	ee
z  dz  ed<   dZee
z  dz  ed<   dZee
z  dz  ed<   dZeed	<   d
Zeed<   dZeed<   dZeed<    fdZ xZS )InklingConfigzETop-level multimodal config (`InklingMMConfig` in the SGLang source).inkling_mm_model)r,   r   r   Nr,   r   r   iv image_token_idiu audio_token_idiE image_bos_token_idiT audio_bos_token_idc                 r   |j                  d      xs i }t        | j                  t              r| j                  j	                  d|j                  d             | j                  j	                  d|j                  dd             | j                  j	                  d|j                  d             t        | j
                  t              r% | j                  d   di | j
                  | _        n%| j
                   | j                  d          | _        t        | j                  t              r% | j                  d	   di | j                  | _        n%| j                   | j                  d	          | _        t        | j                  t              r% | j                  d
   di | j                  | _        n%| j                   | j                  d
          | _        | j                  j                  | j                  _	        | j                  j                  | j
                  _	        t        | ,  di | y )N
mtp_configrv   num_nextn_predict_layersrw   Fry   r]   r   r   r,   r   )r   
isinstancer,   dict
setdefaultr   sub_configsr   rK   r   r   r   )r   r   r   r   s      r   r   zInklingConfig.__post_init__   s   ZZ-3
d&&-''(8*..Ic:de''(@*..QikpBqr''(=z~~N_?`ad''. @ 0 0 @ U4CTCT UD& @ 0 0 @ BDd(($/!B!1!1/!B!XTEWEW!XD'!B!1!1/!B!DDd&&->t//>RAQAQRD%>t//>@D.2.>.>.J.J+-1-=-=-I-I*''r   )r   r   r   __doc__r   r*   r   r   r   r,   r   r   r   r   r   r   r   r   r   r   r   r   s   @r   r   r      s    O#J(*,K 48K"T)D0759L$t+d297;M&-4; NC  NC $$$$( (r   r   c                       e Zd Zy)InklingModelOutputWithPastNr   r   r   r   r   r   r   r   
      r   r   c                       e Zd Zy)InklingCausalLMOutputWithPastNr   r   r   r   r   r     r   r   r   c                       e Zd Zy)InklingRMSNormNr   r   r   r   r   r     r   r   r   c                        e Zd ZdZdedef fdZdej                  dej                  dej                  dej                  fd	Z xZ	S )
InklingRelativeLogitsa|  hidden states conditioned relative position bias. `proj` is a trained bank of bias-vs-distance profiles; each token's
    `relative_states` mixes them into one bias value per backward distance
    (`sglang RelLogitsProj` + the FA4 `score_mod`, materialized densely). The bias is zero
    outside `0 <= distance < rel_extent`; causality and padding stay in the attention mask.
    rX   rZ   c                     t         |           || _        t        j                  t        j                  ||            | _        y N)r   __init__rZ   nn	Parametertorchemptyproj)r   rX   rZ   r   s      r   r   zInklingRelativeLogits.__init__  s0    $LLUJ!?@	r   relative_statesquery_positionskey_positionsreturnc                 j   || j                   z  j                  dd      }|d d d f   |d d d f   z
  d d d d d d f   } |j                  d| j                  dz
        j                  g |j
                  d d dd }|j                  d|      }|j                  |dk  || j                  k\  z  d      S )Nrs   r   r   rn   )r   	transposeclamprZ   expandshapegathermasked_fill)r   r   r   r   
rel_logitsdistancegather_indexposition_biass           r   forwardzInklingRelativeLogits.forward"  s     &		1<<QB
#AtG,}T1W/EEtTSTVWGWXDx~~a1)<=DDcjFVFVWYXYFZc\^c`bc"))"l;(((Q,8t;V)WY\]]r   )
r   r   r   r   r   r   r   Tensorr   r   r   s   @r   r   r     s^    Ac As A
^^ ^ ||	^
 
^r   r   modulequerykeyvaluer9   scalingdropoutr   c                 &   t        || j                        }	t        || j                        }
t        j                  ||	j	                  dd            |z  }|||z   }|||z   }t
        j                  j                  |dt        j                        j                  |j                        }t
        j                  j                  ||| j                        }t        j                  ||
      }|j	                  dd      j                         }||fS )Nr   r   r   )dimdtype)ptrainingrs   )r&   num_key_value_groupsr   matmulr   r   
functionalsoftmaxfloat32tor   r   r   
contiguous)r   r   r   r   r9   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs                r   eager_attention_forwardr  0  s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL #m3!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r   c                        e Zd Zdedef fdZ	 	 ddej                  dej                  dz  dej                  dz  dedz  d	e	e
   d
eej                  ej                  dz  f   fdZ xZS )InklingAttentionconfig	layer_idxc                    t         |           || _        || _        |j                  | j                     dk(  | _        | j
                  r|j                  n|j                  | _        | j
                  r|j                  n|j                  | _
        | j
                  r|j                  n|j                  | _        | j                  | j                  z  | _        | j
                  r|j                  nd | _        | j
                  r|j                  n|j                   | _        d| j                  z  | _        |j$                  | _        d| _        t)        j*                  |j,                  | j                  | j                  z  d      | _        t)        j*                  |j,                  | j                  | j                  z  d      | _        t)        j*                  |j,                  | j                  | j                  z  d      | _        t)        j*                  |j,                  | j                  |j4                  z  d      | _        t)        j*                  | j                  | j                  z  |j,                  d      | _        t;        | j                  | j                  z  |j<                  |d      | _        t;        | j                  | j                  z  |j<                  |d      | _         tC        | j                  |jD                  	      | _#        tC        | j                  |jD                  	      | _$        tK        |j4                  | j                         | _&        y )
Nr{         ?TFbiasr   )conv_idxrs   eps)'r   r   r  r  r^   
is_slidingrW   rS   rT   rO   	num_headsrV   rQ   r   r?   rD   rZ   r   ro   	is_causalr   LinearrK   q_projk_projv_projrX   r_projo_projInklingShortConvolutionrF   k_sconvv_sconvr   r`   q_normk_normr   rel_logits_projr   r  r  r   s      r   r   zInklingAttention.__init__M  sg   " ,,T^^<@PP/3++FOO;???77PVPjPjEI__6#A#AZ`ZtZt $(NNd6N6N$N!<@OOf88QU8<&44VM^M^T]]*!'!9!9ii 2 2DNNT]]4RY^_ii 2 2D4L4Lt}}4\chiii 2 2D4L4Lt}}4\chiii 2 2DNNV\\4QX]^ii >@R@RY^_.$$t}}4f6N6NPYde
 /$$t}}4f6N6NPYde
 %T]]8K8KL$T]]8K8KL4V\\4??Sr   Nr8   r9   	conv_maskpast_key_valuesr   r   c                 H   |j                   d d }g |d| j                  }| j                  |      }| j                  | j	                  |      ||      }	| j                  | j                  |      ||      }
| j                  |      }| j                  |j                  |            j                  dd      }| j                  |	j                  |            j                  dd      }	|
j                  |      j                  dd      }
|j                   d   }|[|j                  || j                        \  }}|j                  | j                        }|j                  |	|
| j                        \  }	}
n|	j                   d   }d\  }}t!        j"                  ||j$                        |z   }t!        j"                  ||j$                        |z   } |j                  g || j&                  d }| j)                  |||      }| j*                  s| j,                  j.                  |dz   j1                         }d| j,                  j2                  t!        j4                  || j,                  j.                  z  j7                  d            z  z   }|j                  dddd      }|j1                         |z  j9                  |j:                        }|j1                         |z  j9                  |j:                        }t=        j>                  | j,                  j@                  tB              } || ||	|
|f| jD                  sd	n| jF                  | jH                  | jJ                  |d
|\  }} |jL                  g |d jO                         }| jQ                  |      }||fS )Nr   r  r  rs   r   )r   r   devicer  )minrn   )r   r   rD   r   ))r   rS   r  r  r  r  r  r  r  viewr   r  get_mask_sizesr  get_query_offsetupdater   aranger#  r  r  r  r  r[   r   r\   logr   r   r   r   get_interface_attn_implementationr  r   ro   r   rD   reshaper   r  )r   r8   r9   r  r  r   input_shapehidden_shapequery_statesr   r   r   q_length	kv_length	kv_offsetq_offsetkv_positionsq_positionsr   effective_ntauattention_interfacer  r   s                           r   r   zInklingAttention.forwardl  s]    $))#2.88b8$--8{{=1\\$++m"<oir\s
||DKK$>P_kt|u++m4{{<#4#4\#BCMMaQRS[[!>?II!QO
#((6@@AF%%a(&#2#A#A(DNN#[ Iy&77GH'6'='=j,X\XfXf'g$J"((+I"&Hi||Im6J6JKiWll8M4H4HIHT./..PPT^^PRP,,_k<X 4;;#B#B#N&?113K55		t{{>>>EE#EN9  C ((1aQ'C(..036::<;M;MNL*002S8<<]=P=PQM(?(M(MKK,,.E)
 %8%
  $}}C$2H2HLL..'%
 %
!\ *k));;;;FFHkk+.L((r   NN)r   r   r   r*   r   r   r   r   r   r   r   tupler   r   r   s   @r   r  r  L  s    T0 TS TF *.(,A)||A) t+A) <<$&	A)
 A) +,A) 
u||U\\D00	1A)r   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )
InklingMLPr  c                     t         |   |       t        |j                     | _        t        j                  t        j                  d            | _	        y Nrs   )
r   r   r   re   act_fnr   r   r   onesglobal_scaler   r  r   s     r   r   zInklingMLP.__init__  s;     V../LLA7r   r8   r   c                     | j                  | j                  | j                  |            | j                  |      z        }|| j                  z  S r   )	down_projr@  	gate_projup_projrB  r   r8   s     r   r   zInklingMLP.forward  sE    t{{4>>-3P'QTXT`T`anTo'opt0000r   )	r   r   r   r*   r   r   r   r   r   r   s   @r   r=  r=    s+    80 8
1U\\ 1ell 1r   r=  c                   $     e Zd Zdef fdZ xZS )InklingExpertsr  c                 h    t         |   |       |j                  | _        |j                  | _        y r   )r   r   r@   num_expertsrf   intermediate_dimrC  s     r   r   zInklingExperts.__init__  s,     !22 & < <r   )r   r   r   r*   r   r   r   s   @r   rJ  rJ    s    =0 = =r   rJ  c                   r     e Zd Z fdZdeej                  ej                  ej                  f   fdZ xZS )InklingTopkRouterc                 T   t         |           |j                  | _        |j                  | _        | j                  | j                  z   | _        |j                  | _        |j                  | _        |j                  | _
        t        j                  t        j                  | j
                  |j                              | _        t        j                  t        j                   d            | _        t        j                  t        j                  | j                              | _        y r?  )r   r   r@   rL  rj   n_total_expertsrK   
hidden_dimrl   ri   top_kr   r   r   r   weightrA  rB  e_score_correction_biasrC  s     r   r   zInklingTopkRouter.__init__  s    !22 & 7 7#//$2G2GG ,,!--//
ll5;;t/C/CVEWEW#XYLLA7')||EKK@P@P4Q'R$r   r   c                    |j                  d| j                        }t        j                  || j                        }|j                         }|dd | j                   f   }|| j                  z   }t        j                  || j                  dd      d   }|dd | j                   f   }|d| j                   d f   }	t        j                  |j                  d|      |	gd      }
t        j                  |
      }t        j                  |t        j                  |dd      z
        }|| j                   z  | j"                  z  }|d| j                   d f   j%                         }|dd | j                  f   j%                         }||||fS )	Nr   .F)r   sortedrs   r   T)r   keepdim)r-  rR  FlinearrT  sigmoidrj   rU  r   topkrS  catr   
logsigmoidexp	logsumexprl   rB  r   )r   r8   flatrouter_logitsscoresrouted_scoresscores_for_choicetopk_indicesrouted_logitsshared_logitstopk_logitstopk_log_probstopk_weightsshared_gammass                 r   r   zInklingTopkRouter.forward  s   $$R9t{{3 &&(s$<t'<'<&<$<<=)D,H,HHzz"3TZZRPUVWXY%c+Cd.C.C-C+C&CD%cD,A,A+A+C&CDii!5!5b,!G W]_`k2yy%//.VXbf2g!gh#d&6&669J9JJ$S4+@+@*@*B%BCNNP#C4::$56AAClL-GGr   )	r   r   r   r   r;  r   r   r   r   r   s   @r   rO  rO    s/    SHellELL%,,.V(W Hr   rO  c                   $     e Zd Z fdZd Z xZS )InklingSharedExpertsc                    t         |           |j                  | _        |j                  }t	        j
                  t        j                  |j                  ||j                              | _	        t	        j
                  t        j                  |j                  ||j                              | _
        t	        j
                  t        j                  |j                  |j                  |            | _        t        |j                     | _        y r   )r   r   rj   rf   r   r   r   r   rK   rF  rG  rE  r   re   r@  )r   r  rM  r   s      r   r   zInklingSharedExperts.__init__  s     & 7 7!77
 ekk&2I2IK[]c]o]o&pq||EKK0G0GIY[a[m[m$noekk&2I2I6K]K]_o&pqV../r   c                    |j                   }|j                  dd|d         j                  | j                  dd      }|j                  d| j                  d      j	                  dd      }t        j                  || j                  j	                  dd            }t        j                  || j                  j	                  dd            }| j                  |      |z  |z  }t        j                  || j                  j	                  dd            }|j                         j                  d      j                  |j                        }|j                  |      S )Nrs   r   r   r   rX  )r   r-  r   rj   r   r   bmmrF  rG  r@  rE  r   sumr   r   r%  )	r   r8   gammasr.  gateup	activateddownouts	            r   r   zInklingSharedExperts.forward  s   #))%--a[_ELLTMbMbdfhjkD$9$91=GG1Myy(@(@A(FGYY}dll&<&<Q&BCKK%*V3	yyDNN$<$<Q$BCjjl1%(()<)<=xx$$r   )r   r   r   r   r   r   r   s   @r   ro  ro    s    0%r   ro  c                   B     e Zd ZdZ fdZdej                  fdZ xZS )
InklingMoEz7Gate -> routed experts (+ shared experts), TML flavour.c                     t         |           || _        t        |      | _        t        |      | _        t        |      | _        y r   )	r   r   r  rO  ru  rJ  expertsro  shared_expertsrC  s     r   r   zInklingMoE.__init__  s:    %f-	%f-26:r   r   c                     |}|j                   }| j                  |      \  }}}}|j                  d|j                   d         } | j                  |||      j                  | }|| j	                  ||      z   }|S )Nr   )rt  )r   ru  r%  r}  r~  )r   r8   	residualsr.  _rl  rg  rm  s           r   r   zInklingMoE.forward  s    !	#))7;yy7O4<}%**2}/B/B2/FGT]L,OTTVab%(;(;Im(;(\\r   )	r   r   r   r   r   r   r   r   r   r   s   @r   r{  r{    s    A; r   r{  causal_conv1d_updater8   
conv_staterT  r
  
activationc                    | j                   \  }}}|j                   d   }t        j                  || gd      j                  |j                        }	|j                  |	d d d d | d f          t        j                  |	|j                  d      |d|      }
|
d d d d | d f   }
|t        |   |
      }
|
j                  | j                        S )Nr   rX  rs   r   )paddinggroups)
r   r   r^  r   r   copy_rZ  conv1d	unsqueezer   )r8   r  rT  r
  r  r  rK   seq_len	state_lenhidden_states_newry  s              r   r  r    s     ,11A{G  $I		:}"=2FII&,,W&q!iZ['89:
(($f&6&6q&94S^
_C
aWHIo
CZ %66-%%&&r   causal_conv1d_fnc                 8   | j                   \  }}}|j                   d   dz
  }t        j                  | j                  |j                        |j                  d      |||      d d d d d |f   }	|t        |   |	      }	|	j                  | j                        S )Nr   rs   )rT  r
  r  r  )r   rZ  r  r   r   r  r   )
r8   rT  r
  r  r   r  rK   r  r  ry  s
             r   r  r  ,  s     ,11A{Gll2"G
((&" HWHnC Z %66-%%&&r   c                        e Zd Zdedededef fdZ ed      	 	 ddej                  d	edz  d
ej                  dz  de	e
   fd       Z xZS )r  rK   rA   r  r  c                     t         |           || _        || _        || _        t        j                  |||||dz
  d      | _        y )Nrs   F)in_channelsout_channelskernel_sizer  r  r
  )r   r   r  r  rA   r   Conv1dr  )r   rK   rA   r  r  r   s        r   r   z InklingShortConvolution.__init__E  sK    "  0ii#$($q(
r   r  Nr8   r  r  r   c                    |j                   }|j                         }|}t        ||      }|j                  d   }|j	                  dd      }|d uxr& |j                  | j                  | j                        }|r|dk(  r|j                  | j                     j                  sv|j                  | j                     j                  | j                     }	t        ||	| j                  j                  j                  d      | j                  j                        }n|3|j!                  || j                  | j                  | j"                        }t%        || j                  j                  j                  d      | j                  j                  |j'                  d            }|r|d d d d | d f   }|j	                  dd      }||z   j)                  |      }|S )Nrs   r   )	state_idxrA   seq_idx)r  )r   )r   r   r(   r   r   has_previous_stater  r  r:   record_pastconv_statesr  r  rT  squeezer
  update_conv_staterA   r  r   r   )
r   r8   r  r  r   input_dtyperesidualr  use_precomputed_statesr  s
             r   r   zInklingShortConvolution.forwardT  s    $))%++- 4]IN%%a(%//15!0!< "
AcAcNNDMMB
 "gl?;Q;QRVR`R`;a;m;m(//?KKDMMZJ0z4;;+=+=+E+Ea+H$++JZJZM * / A A!4>>T]]]a]r]r !B ! -t{{1199!<dkk>N>NX^XbXbclXmM
 & -aWHIo >%//15&155K5Hr   r:  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   s   @r   r  r  C  s    
C 
3 
3 
Z] 
 H% )-)-	*||* * <<$&	*
 +,* &*r   r  c                        e Zd Zdedef fdZ	 	 	 ddej                  dej                  dz  dej                  dz  dedz  d	e	e
   d
ej                  fdZ xZS )InklingDecoderLayerr  r  c                 (   t         |           |j                  | _        t        ||      | _        |j
                  |   dk(  rt        |      | _        nt        |      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        |j                  |   | _        t        |j                  |j                   |d      | _        t        |j                  |j                   |d      | _        y )Nr   r   )r  r  r   )r   r   rK   r  	self_attnrb   r{  mlpr=  r   r`   input_layernormpost_attention_layernormr^   
layer_typer  rA   
attn_sconv	mlp_sconvr  s      r   r   zInklingDecoderLayer.__init__  s    !--)&)<!!),8!&)DH!&)DH-f.@.@&BUBUV(6v7I7I6K^K^(_% ,,Y71 7 79WX
 1 7 79WX
r   Nr8   r9   r  r  r   r   c                    |}| j                  |      } | j                  d||||d|\  }}| j                  |||      }||z   }|}| j                  |      }| j	                  |      }| j                  |||      }||z   }|S )N)r8   r9   r  r  r!  r   )r  r  r  r  r  r  )r   r8   r9   r  r  r   r  r  s           r   r   zInklingDecoderLayer.forward  s     !,,];)4>> 
')+	

 
q bkl =0 55mD/}oajk =0r   )NNN)r   r   r   r*   r   r   r   r   r   r   r   r   r   r   s   @r   r  r    s    
0 
S 
. /3)-(,|| t+ <<$&	
  +, 
r   r  c                        e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdgZg dZeedZ ej(                          fd	       Z xZS )
InklingPreTrainedModelmodelTr  r  Fzmodel\.mtp\..*)r  r  r  r  )r8   
attentionsc                    t         |   |       | j                  j                         j                  }t        |t              r#t        j                  |j                  d|       y t        |t              r t        j                  |j                         y t        |t              rEt        j                  |j                  d|       t        j                  |j                  d|       y t        |t               rat        j                  |j"                  d|       t        j                  |j                         t        j$                  |j&                         y t        |t(              rgt        j                  |j*                  d|       t        j                  |j,                  d|       t        j                  |j                  d|       y t        |t.              rlt1        | j                  d| j                        }t        j2                  |j4                  t7        j8                  |j:                        |j<                  z         y y )Nrn   )meanstdr   )r   _init_weightsr  get_text_configrq   r   r   initnormal_r   r=  ones_rB  rJ  gate_up_projrE  rO  rT  zeros_rU  ro  rF  rG  InklingAudioModelEmbeddingsgetattrr  audio_tokens_offsetsr   r)  r   r   )r   r   r  r   r   s       r   r  z$InklingPreTrainedModel._init_weights  s   f%kk))+==f34LL3C8
+JJv**+/LL,,3C@LL))= 12LLSc:JJv**+KK667 45LL))=LLcs;LL))= ;< #4;;LLJJ++\4458S8SS	 =r   )r   r   r   r   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backend"_keys_to_ignore_on_load_unexpected_keep_in_fp32_modules_strictr  r  _can_record_outputsr   no_gradr  r   r   s   @r   r  r    s~     L&*#./#4"5 !N""'*;)<&#T ,&
 U]]_ r   r  c                        e Zd ZU eed<   def fdZeee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  d	edz  d
ee   defd                     Z xZS )InklingTextModelr  c           	      6   t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        d| _        | j'                          y c c}w )Nr  F)r   r   rr   padding_idxrH   r   	EmbeddingrK   r2   
ModuleListr   rM   r  r:   r   r`   r;   
embed_normgradient_checkpointing	post_initr  s      r   r   zInklingTextModel.__init__  s     !.. ++LL):):F<N<NPTP`P`ammEJ6KcKcEde	 3e
 #6#5#56;N;NO	(););ATATU&+# 	 fs   DNr6   r9   position_idsr  r7   	use_cacher   r   c                    |d u |d uz  rt        d      | | j                  | j                  |            }|r|t        | j                        }|V||j                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s3| j                  ||||d}
t        di |
t        di |
t        di |
d}	|}t!        | j"                        D ]8  \  }}| j                  j$                  |   dk(  rd	nd
} ||f|	|   |	d   |d|}: | j'                  |      }t)        ||      S )N:You must specify exactly one of input_ids or inputs_embeds)r  r   rs   r"  r  r7   r9   r  r  full_attentionsliding_attentionlinear_attentionr|   r  r  r  )r9   r  r  )last_hidden_stater  r   )
ValueErrorr  r2   r	   r  get_seq_lengthr   r)  r   r#  r  r   r   r   r   r   	enumerater:   r^   r;   r   )r   r6   r9   r  r  r7   r  r   past_seen_tokenscausal_mask_mappingmask_kwargsr8   r   decoder_layerattention_types                  r   r   zInklingTextModel.forward  s    -t";<YZZ  OOD,=,=i,HIM0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L ?-F++!."0#2 ,K #5"C{"C%F%U%U$C$Rk$R# & )$++ 6 	A}151H1H1Kx1W-]pN)2>B-.@A /	
 M	 		-0&++
 	
r   )NNNNNN)r   r   r   r*   r   r   r   r   r   r   
LongTensorr   r   FloatTensorr   r   r   r   r   r   r   s   @r   r  r    s    0     .2.204(,26!%6
##d*6
 t+6
 &&-	6

 6
 ((4/6
 $;6
 +,6
 
!6
    6
r   r  c                      e Zd Zi ZddiZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
edz  deej                  z  dee   defd              Zy)InklingForCausalLMlm_headrowwise_split_inputNr6   r9   r  r  r7   labelsr  logits_to_keepr   r   c	           
          | j                   d||||||d|	}
|
j                  | j                  j                  z  }t	        |t
              rt        | d       n|}| j                  |d d |d d f         }| j                  j                  }|||j                  d   k  r	|dd |f   }d }|# | j                  d|||j                  d   d|	}t        |||
j                  |
j                  |
j                        S )N)r6   r9   r  r  r7   r  r   .logitsr  rH   )lossr  r  r8   r  r   )r  r  r  r>   r   r   slicer  rI   r   loss_functionr   r  r8   r  )r   r6   r9   r  r  r7   r  r  r  r   outputsr8   slice_indicesr  rI   r  s                   r   r   zInklingForCausalLM.forward:  s&    $** 
)%+'
 
  11DKK4[4[[8B>SV8W~ot4]kmA}a,?@A"kk==*/BV\\RTEU/UC!5"5!556F%4%%jVFv||\^O_jcijD,#33!//))
 	
r   )NNNNNNNr   )r   r   r   _tied_weights_keys_tp_planr   r   r   r  r   r   r  r   r   r   r   r   r   r   r   r   r  r  5  s    01H .2.204(,26*.!%-.(
##d*(
 t+(
 &&-	(

 (
 ((4/(
   4'(
 $;(
 ell*(
 +,(
 
'(
  (
r   r  c                       e Zd Zy)r  Nr   r   r   r   r  r  g  s    r   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )InklingAudioModelr  c                 |    t         |   |       t        |      | _        t	        |j
                  d      | _        y )Nr_   r  )r   r   r  embed_audio_tokensr   r   r;   rC  s     r   r   zInklingAudioModel.__init__k  s1     "=f"E"6#:#:E	r   audio_input_idsr   c                 `    | j                  |      }| j                  |      }t        ||      S )Nr  pooler_output)r  r;   r   )r   r  r8   s      r   r   zInklingAudioModel.forwardp  s3    //@		-0)+'
 	
r   )	r   r   r   r   r   r   r   r   r   r   s   @r   r  r  j  s-    F1 F

u|| 
 
r   r  c            
            e Zd Zdededededef
 fdZdej                  dej                  fd	Zdej                  dej                  fd
Z	 xZ
S )InklingVisionEncoderLayer	input_dim
output_dimt_foldhw_foldadd_normc                     t         |           t        j                  ||d      | _        |rt        |      | _        || _        || _        || _	        y NFr	  )
r   r   r   r  
projectionr   
layer_normr  r  r  )r   r  r  r  r  r  r   s         r   r   z"InklingVisionEncoderLayer.__init__z  sF    ))IzF,Z8DO r   r8   r   c           
         |j                   \  }}}}}|| j                  z  }|| j                  z  }|| j                  z  }	|j                  ||| j                  || j                  |	| j                  |      }|j	                  dddddddd      }|j                  ||||	| j                  | j                  z  | j                  z  |z        }|S )	z
        Convert a tensor of shape (B, T, H, W, C) to a tensor of shape (B, T // t, H // hw, W //  hw, C * (t * hw**2))
        r   rs   r      r   ra   rh      )r   r  r  r-  permute)
r   r8   BTHWCt_newh_neww_news
             r   fold_timespace_to_depthz1InklingVisionEncoderLayer.fold_timespace_to_depth  s     &++1aAT[[ T\\!T\\!%--aUDLLZ_aeamamopq%--aAq!Q1E%--audkkTXT`T`F`cgcocoForsFstr   c                     | j                   dkD  s| j                  dkD  r| j                  |      }| j                  |      }| j                  r&| j                  |      }t        j                  |      }|S r?  )r  r  r  r
  r  r  rZ  gelurH  s     r   r   z!InklingVisionEncoderLayer.forward  s_    <<!t{{Q 88GM6== OOM:MFF=1Mr   )r   r   r   r   r   r   r   r   r  r   r   r   s   @r   r  r  y  sb    !# !3 ! !c !]a !U\\ ell  U\\ ell r   r  numberr   c                 $   g }| dz  dk(  r|j                  d       | dz  } | dz  dk(  rt        dt        j                  |       dz   d      D ]*  }| |z  dk(  s|j                  |       | |z  } | |z  dk(  r, | dkD  r|j                  |        |S )Nr   r   r   rs   )appendr   mathisqrt)r  factorsr   s      r   prime_factorsr!    s    G
1*/q1 1*/ 1djj(1,a0 qjAoNN1qLF qjAo
 zvNr   r   r   r   
n_channelsc           	         t        j                  t        j                  t        |      ddd   |      d      }t        j                  t        j                  t        |       ddd   |      d      }t        j                  |dz  |z  dz        j                         dz  }t        j                  |d   dz  |z  |z        j                         dz  }t        j                  ddd|gg|      }	t        j                  t        j                  |      |||gd      }
t        j                  |t        j                  ||d         t        j                  ||d         |gd      }t        j                  |	|
|gd      }t        j                  |ddddf   d      j                         }||z  | z  |z  }t        j                  dt        j                  t        j                  ||            |dz   |      }t        j                  |j                  d      t        j                  |      j                  d      z
        }||j                   d   k\  rt        j"                  |d      }nDdd	lm}  ||j)                         j+                               \  }}t        j                  ||      }d|d<   |j                   d   dz
  |d<   ||   S )
a  
    Plan out the dimensions for each layer in the HMLP encoder.

    This function determines the progression of dimensions (temporal, height, width, channels)
    for a multi-layer perceptual model that processes image/video patches. It follows these
    principles:
    1. Start with small dimensions and increase to full size
    2. Expand spatial dimensions (height/width) first, then temporal
    3. Increase channel count to avoid information bottlenecks
    4. Round channel dimensions to multiples of 64 for hardware efficiency

    The function computes optimal assignments of scale configurations to layers using either:
    - For n_layers >= len(scales): Individual best matching scales for each layer (allowing duplicates)
    - For n_layers < len(scales): Global optimal assignment via linear_sum_assignment

    The first and last scales are always fixed to ensure the proper input and output dimensions.

    Args:
        temporal_patch_size: Temporal dimension of input patches
        patch_size: Spatial dimension (height/width) of input patches
        n_layers: Number of layers in the encoder
        n_channels: Number of input channels (default: 3 for RGB)

    Returns:
        torch.LongTensor of shape `(n_layers + 1, 4)` where the last dim holds values for (t, h, w, c) grids.
    Nr   r"  r   rX  r   rN   rs   )linear_sum_assignment)r   cumprodtensorr!  ceilr   stack	ones_like	full_liker^  prodr   linspacer*  absr  r   argminscipy.optimizer$  cpunumpy)r   r   r   r"  r#  hth_cht_chbasespatialtemporalscalessize_reductiontotal_elementslog_ideal_scalescost_matrixidxsr$  r  idxs_nps                        r   plan_out_scalesr@    s]   : 	ell=#<TrT#B6RXYZAell=1D#Edd#KTZ[abcA::adZ'",-113b8D::aeqj:-126682=D<<!Q:./?Dkk5??1-q!T:BG{{Auq!B%8%//!QrU:SUYZ`abHYYgx0a8FZZq#2#vA6<<>N*,/BBZON~~	599U\\.@A8a<X^ )),66q9EIIn<U<_<_`a<bbcK6<<?"||KQ/8*;??+<+B+B+DE
7||GF3 DG||A"DH$<r   c                   f     e Zd Zdef fdZdej                  dee   dej                  fdZ	 xZ
S )InklingVisionModelr  c                    t         	|   |       t        |j                  |j                  |j
                  |j                        | _        t        j                         | _
        t        t        | j                  d d | j                  dd              D ]  \  }\  }}|d   |d   z  |d   |d   z  z  |d   |d   z  z  }||j
                  dz
  k(  r|j                  n|d   }|d   |d   z  }|d   |d   z  }| j                  j                  t        |d   |z  |||||j
                  dz
  k7                t!        |j                        | _        | j%                          y )Nr   rs   r   r   r   )r  r  r  r  r  )r   r   r@  r   r   rM   r   r9  r   r  encoder_layersr  zipr   r  r  r   
final_normr  )
r   r  r   start_scale	end_scaleshuffle_multr  r  r  r   s
            r   r   zInklingVisionModel.__init__  s{    %&&$$	
 !mmo+4SSb9I4;;WXWY?5[+\ 	'A'Y1Q/IaLKPQN4RSW`abWcgrstguWuv  569Q9QTU9U4U00[def[gJlk!n4Gq\[^3F&&))!n|;)#!&":":Q">>	" ))@)@Ar   pixel_valuesr   r   c                     |j                   d   }|}| j                  D ]  } ||      } | j                  |      }|j                  |d      }t	        ||      S )Nr   )r8   r   r  )r   rD  rF  r-  r   )r   rJ  r   num_patchesr8   layers         r   r   zInklingVisionModel.forward  sk    "((+$(( 	?E!>M	? 6%--k2>)+'
 	
r   )r   r   r   r   r   r   r   r   r   r   r   r   s   @r   rB  rB    s;    2 >
ELL 
FCU<V 
[`[g[g 
r   rB  zz
    The Base Inkling model which consists of a vision backbone and a language model without language modeling head.,
    custom_introc                       e Zd ZdZdef fdZe ed      dej                  de
e   deez  fd	              Ze ed
      	 ddej                  dej                   dz  deez  fd              Zdej                  dej                  dej                  defdZee	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                   dz  dej                   dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  de
e   deez  fd              Z xZS )InklingModelFr  c                    t         |   |       |j                  j                  | _        t	        |j                        | _        t        |j                        | _        t        |j                        | _        | j                          y r   )r   r   r,   rH   r  language_modelr  r   audio_towerrB  r   vision_towerr  rC  s     r   r   zInklingModel.__init__$  sf      ,,77.v/A/AB,V-@-@A.v/C/CDr   zOProjects the last hidden state from the vision model into language model space.rN  rJ  r   r   c                 *     | j                   dd|i|S )NrJ  r   )rU  r   rJ  r   s      r   get_image_featureszInklingModel.get_image_features,  s    
 !t  ElEfEEr   zCProjects discretized dMel bin tokens into the language model space.Nr  audio_input_ids_maskc                     |||j                            }n|j                  d|j                  d         }| j                  |      S )a  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
            Batch of (padded) dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) frames. When provided, only valid frames are encoded so that the
            number of returned audio embeddings matches the number of audio placeholder tokens.
        r   )r   r-  r   rT  )r   r  rY  s      r   get_audio_featureszInklingModel.get_audio_features3  sM      +-.B.G.G.IJO-55b/:O:OPR:STO00r   r6   r7   featurestoken_idc                    |Y| | j                         t        j                  |t        j                  |j                              k(  }|j                  d      }n||k(  }|j                         }|j                  d      j                  |      j                  |j                        }t        ||   j                         |j                         k(  d| d|j                  d           |S )z
        Obtains a multimodal placeholder mask from `input_ids` or `inputs_embeds` for the given `token_id`, and checks
        that the placeholder token count matches the length of `features`. If the lengths differ, an error is raised.
        )r   r#  r   zAMultimodal features and placeholder tokens do not match, tokens: z, features: r   )get_input_embeddingsr   r&  longr#  allrs  r  	expand_asr   r   numelr   )r   r6   r7   r\  r]  special_maskn_tokenss          r   get_placeholder_maskz!InklingModel.get_placeholder_maskG  s     (,GD,E,E,GXUZZ@T@TU- L (++B/L$0L##%#--b1;;MJMMmNbNbc,'--/8>>3CCOPXzYefnftftuvfwexy	
 r   r9   r  r  token_type_idsr  r  	lm_kwargsc           	         |du |	duz  rt        d      |	/| j                  j                   | j                         |            }	|{| j	                  |      j
                  }|j                  |	j                  |	j                        }| j                  ||	|| j                  j                        }|	j                  ||      }	d}||| j                  ||      j                  }|j                  |	j                  |	j                        }| j                  ||	|| j                  j                        }|	j                  ||      }	t!        |x}t"              sA| j                  j%                         |	|||d}t'        di |t)        di |t+        di |d} | j                  d||||	|d|}t-        |j                  |j.                  |j0                  |j2                  |      S d      S )a/  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("google/inkling2-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/inkling2-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr  r  r  )r9   r  r  r7   r  )r  r  r8   r  image_hidden_statesr   )r  rS  r  r_  rX  r   r   r#  r   rf  r  r   masked_scatterr[  r  r   r   r   r  r   r   r   r   r  r8   r  )r   r6   rJ  r  rY  r9   r  r  rg  r7   r  r  rh  image_featuresspecial_image_maskaudio_featuresspecial_audio_maskr  r  r  s                       r   r   zInklingModel.forwardb  s!   d -t";<YZZ  //::;V4;T;T;VW`;abM #!44\BPPN+..}/C/C]EXEXYN!%!:!:=.$++:T:T" *889K^\M &!44_FZ[mmN+..}/C/C]EXEXYN!%!:!:=.$++:T:T" *889K^\M ?-F++557!."0#2 ,K #5"C{"C%F%U%U$C$Rk$R# &$%% 
.%+'
 
 *%77#33!//))2>2J
 	

 QU
 	
r   r   )NNNNNNNNNNN)r   r   r   accepts_loss_kwargsr   r   r   r   r   r  r   r   r;  r   rX  r  r   r[  r   rf  r   r   r   r   r   r   s   @r   rQ  rQ    s@     }  !rsF!--F9?@R9SF	+	+F t F
 !fg 591))1 $llT11 
+	+	1 h 1$## (( ##	
 6  .2153748.204(,2626*.!%h
##d*h
 ''$.h
 ))D0	h

 $llT1h
 t+h
 &&-h
 h
 ((4/h
 ((4/h
   4'h
 $;h
 ./h
 
+	+h
  h
r   rQ  c                       e Zd Zi ZddiZdZdef fdZede	j                  dee   fd       Zee	 	 	 	 	 	 	 	 	 	 	 dd
e	j                  d	z  de	j                  d	z  de	j                   d	z  de	j                  d	z  ded	z  de	j                  d	z  de	j                   d	z  de	j                  d	z  de	j                  d	z  ded	z  dee	j                   z  dee   deez  fd              Z	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )InklingForConditionalGenerationr  r  Fr  c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y r	  )r   r   rQ  r  r   r  r,   rK   rH   r  r  rC  s     r   r   z(InklingForConditionalGeneration.__init__  sU     !&)
 yy!3!3!?!?ASASA^A^ejkr   rJ  r   c                 <     | j                   j                  |fi |S r   )r  rX  rW  s      r   rX  z2InklingForConditionalGeneration.get_image_features  s    ,tzz,,\DVDDr   Nr6   r9   r  r  r  rY  r7   r  r  r  r   c                 6    | j                   d|||||||||
|	d
|}|d   | j                  j                  j                  z  }t	        |t
              rt        | d      n|}| j                  |dd|ddf         }| j                  j                  j                  }|||j                  d   k  r	|dd|f   }d}|	# | j                  d||	|j                  d   d|}t        |||j                  |j                  |j                  |j                        S )	a	  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        )
r6   rJ  r  rY  r9   r  r  r7   r  r  r   Nr   .r  )r  r  r  r8   r  rj  r   )r  r  r,   r>   r   r   r  r  rI   r   r  r   r  r8   r  rj  )r   r6   rJ  r9   r  r  r  rY  r7   r  r  r  r   r  r8   r  r  rI   r  s                      r   r   z'InklingForConditionalGeneration.forward  sF   D $** 
%+!5)%+'
 
  
T[[%<%<%X%XX8B>SV8W~ot4]kmA}a,?@A"kk55II*/BV\\RTEU/UC!5"5!556F%4%%jVFv||\^O_jcijD,#33!//)) ' ; ;
 	
r   c                 ^    t        |   |f|||||	|
|d|}|s|	s||d<   ||d<   ||d<   |S )N)r  r7   r9   r  r  r  is_first_iterationrJ  r  rY  )r   prepare_inputs_for_generation)r   r6   r  r7   r  rJ  r9   r  rY  r  r  r  rw  r   model_inputsr   s                  r   rx  z=InklingForConditionalGeneration.prepare_inputs_for_generationM  sh    " w<

+')%)1

 

 Y+7L(.=L*+3GL/0r   )NNNNNNNNNNr   )NNNNNNNTNNF)r   r   r   r  r  rp  r   r   r   r   r  r   r   rX  r   r  r   r   r   r   r;  r   r   rx  r   r   s   @r   rr  rr    s    01H  }  Eu/@/@ EFSeLf E E  .215.204(,374826*.!%-.a
##d*a
 ''$.a
 t+	a

 &&-a
 a
 ))D0a
 $llT1a
 ((4/a
   4'a
 $;a
 ell*a
 +,a
 
.	.a
  a
L ! " "r   rr  )r   r*   r   r   r  r  r  r  rB  rQ  rr  )rn   Nr:  )r0  )lr  collections.abcr   r   torch.nnr   torch.nn.functionalr   rZ  huggingface_hub.dataclassesr    r   r  activationsr   cache_utilsr   r	   configuration_utilsr
   
generationr   integrationsr   r   integrations.accelerater   masking_utilsr   r   r   modeling_layersr   modeling_outputsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   utils.output_capturingr   gemma3.modeling_gemma3r    r!   r"   r#   &higgs_audio_v2.modeling_higgs_audio_v2r$   llama.modeling_llamar%   r&   mixtral.modeling_mixtralr'   qwen3_next.modeling_qwen3_nextr(   
get_loggerr   loggerr*   r   r   r   r   r   r   Moduler   r   r   r  r  r=  rJ  rO  ro  r{  r   r   r  r  r  r  r  r  r  r  r  r  r   r   r!  r  r@  rB  rQ  rr  __all__r   r   r   <module>r     s7     $     . & ! . 3 ) I = s s 9 S F &  8 5  L : 5 I 
		H	% y( y yx $) $ $  $* $ $  +($ +( +(\	!: 		$@ 		\ 	^BII ^B )-%II%<<% 
% <<	%
 LL4'% % % <<$&%8a)ryy a)H1 1=^ =#H		 #HL%299 %8 ( 01
 !%!'<<'' LL' ,,
	'
 d
' 2'& ,- !%!	'<<'LL' ,,
' d
	' .', *,<=>;bii ; ?;|.4 .b ._ . .b L
- L
 L
^/
* /
d ?"8 >
. 
"		 "J# $s) $ W\;;*-;9<;JM;
;|+
/ +
\ 
l
) l

l
^ 
[&<o [
[|r   