Ë
    ÛÓ^j±é  ã                   óä  — d dl Z d dlmZ d dlZd dlmZ d dlmc mZ d dl	m
Z
 ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z? ddl@mAZA  e.j„                  eC«      ZDe
 G d„ de«      «       ZEe
 G d„ de«      «       ZFe
 G d„ de«      «       ZGe
 G d „ d!e«      «       ZH G d"„ d#e8«      ZI G d$„ d%e5«      ZJ G d&„ d'e<«      ZK G d(„ d)ej˜                  «      ZM	 	 did*ej˜                  d+ejœ                  d,ejœ                  d-ejœ                  d.ejœ                  dz  d/eOd0eOd1ejœ                  dz  fd2„ZP G d3„ d4ej˜                  «      ZQ G d5„ d6e7«      ZR G d7„ d8e?«      ZS G d9„ d:ej˜                  «      ZT G d;„ d<ej˜                  «      ZU G d=„ d>ej˜                  «      ZV ed?«      	 	 djd@ejœ                  dAejœ                  dBej®                  dCej®                  dz  dDeXdz  f
dE„«       ZY edF«      	 	 djd@ejœ                  dBej®                  dCej®                  dz  dDeXdz  fdG„«       ZZ eeYeZg«       G dH„ dIej˜                  «      «       Z[ G dJ„ dKe!«      Z\e, G dL„ dMe'«      «       Z]e, G dN„ dOe]«      «       Z^ G dP„ dQe6«      Z_ G dR„ dSe:«      Z` G dT„ dUe]«      Za G dV„ dWej˜                  «      ZbdXecdYedec   fdZ„Ze	 dkd[ecd\ecd]ecd^ecdYejÌ                  f
d_„Zg G d`„ dae]«      Zh e,db¬c«       G dd„ dee]«      «       Zi e,db¬c«       G df„ dge]e«      «       Zjg dh¢Zky)lé    N)ÚCallable)Ústricté   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚGemma3CausalLMOutputWithPastÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3ModelOutputWithPast)ÚHiggsAudioV2Embeddings)ÚLlamaRMSNormÚ	repeat_kv)ÚMixtralExperts)Úapply_mask_to_padding_statesc                   ó,  ‡ — e Zd ZU dZdZddddddddddddœZd	gd
gfddgdgfdgdgfdœZdddddœZddddddœZdZ	e
ed<   dZe
dz  ed<   dZe
ed<   dZe
ed<   dZe
ed <   d!Ze
ed"<   d#Ze
ed$<   dZe
ed%<   d&Ze
ed'<   d#Ze
ed(<   d)Ze
ed<   d&Ze
ed*<   d+Ze
ed,<   dZe
dz  ed-<   d.Zeed/<   dZee
   dz  ed0<   dZee   dz  ed1<   d2Ze
ed<   d3Z eed4<   d5Z!e
ed<   dZ"ee   dz  ed6<   d7Z#e
ed8<   d9Z$eed:<   d;Z%e
ed<<   d=Z&e
ed<   d>Z'e
ed?<   d@Z(e
edA<   dBZ)e*edC<   dDZ+eedE<   dFZ,eed<   d3Z-eedG<   dHZ.eedI<   dJZ/eedK<   dZ0e
dz  edL<   dMZ1e
dz  edN<   d@Z2e
dz  edO<   dZ3e
dz  edP<   dQZ4e*edR<   dBZ5e*edS<   dZ6ee
   dz  edT<   ˆ fdU„Z7e8dV„ «       Z9e8dW„ «       Z:ˆ xZ;S )XÚInklingTextConfigÚinkling_textÚtext_configÚembedding_rowwiseÚcolwiseÚrowwiseÚmoe_tp_expertsÚ
all_reduce)Úembed_tokensú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)r2   ÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater3   r4   r5   Úlogits_mup_width_multiplierÚsliding_window_sizeÚn_routed_expertsÚconv_kernel_sizeÚmax_position_embeddings)Úembedding_multiplierÚsliding_windowÚnum_local_expertsÚsconv_kernel_sizeÚmodel_max_lengthi@ Ú
vocab_sizeNÚunpadded_vocab_sizeé   Úhidden_sizeéB   Únum_hidden_layersé@   Únum_attention_headsé   Únum_key_value_headsé€   Úhead_dimÚswa_num_attention_headsé   Úswa_num_key_value_headsÚswa_head_dimi   Úd_relé   Ú
rel_extentÚlog_scaling_n_floorgš™™™™™¹?Úlog_scaling_alphaÚlocal_layer_idsÚlayer_typesi   ç�íµ ÷Æ°>Úrms_norm_epsé   Úmlp_layer_typesi `  Úintermediate_sizeÚsiluÚ
hidden_acti   Úmoe_intermediate_sizeé   é   Únum_experts_per_tokr   Ún_shared_expertsTÚshared_expert_sinkg       @Úroute_scaleg      8@Úrms_norm_eps_moe_gateç        Úattention_dropoutç{®Gáz”?Úinitializer_rangeÚpad_token_idé   Úbos_token_idÚeos_token_idÚnum_mtp_layersFÚchain_hidden_post_normÚmtp_hidden_states_firstÚmtp_local_layer_idsc                 óF  •— | j                   €{| j                  �t        | j                  «      }n+t        | j                  «      D �ch c]  }|dz   dz  sŒ|’Œ }}t        | j                  «      D �cg c]
  }||v rdnd‘Œ c}| _         | j
                  €A|j                  dd«      }t        | j                  «      D �cg c]  }||k  rdnd‘Œ c}| _        |j                  d	«      �|j                  d	«      | _        d
| _	        t        ‰| �,  di |¤Ž y c c}w c c}w c c}w )Nrs   rh   Úhybrid_slidingÚhybridÚdense_mlp_idxr   ÚdenseÚsparseÚdense_intermediate_sizera   © )r^   r]   ÚsetÚrangerM   rb   ÚpopÚgetrc   Únumber_of_conv_statesÚsuperÚ__post_init__)ÚselfÚkwargsr]   Úir}   Ú	__class__s        €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/inkling/modular_inkling.pyrˆ   zInklingTextConfig.__post_init__‘   s  ø€ Ø×ÑÐ#Ø×#Ñ#Ð/Ü"% d×&:Ñ&:Ó";‘ä.3°D×4JÑ4JÓ.KÖ"[¨ÐPQÐTUÑPUÐYZË{¢1Ð"[�Ð"[äPUÐVZ×VlÑVlÓPmö ØKL A¨Ñ$8Ñ ¸hÑFò ˆDÔð ×ÑÐ'Ø"ŸJ™J ¸Ó:ˆMÜX]Ð^b×^tÑ^tÓXuÖ#vÐST¨q°=Ò/@¡GÀhÑ$NÒ#vˆDÔ à�:‰:Ð/Ó0Ð<Ø%+§Z¡ZÐ0IÓ%JˆDÔ"ð &'ˆÔ"ä‰ÑÑ' Ó'ùò #\ùò ùò
 $ws   ÁDÁDÁ2DÂ>Dc                 óÄ   — | j                   �O| j                  €dg| j                   z  S t        | j                   «      D �cg c]  }|| j                  v rdnd‘Œ c}S y c c}w )Nr|   r{   )rv   ry   rƒ   )r‰   r‹   s     r�   Úmtp_layer_typesz!InklingTextConfig.mtp_layer_types¦   sr   € à×ÑÐ*Ø×'Ñ'Ð/Ø �z D×$7Ñ$7Ñ7Ð7ô ^cÐcg×cvÑcvÓ]wöØXY¨¨T×-EÑ-EÑ(EÑ$È8ÑSòð ð ùòs   Á Ac                 ó<   — | j                   �dg| j                   z  S y )Nr~   )rv   )r‰   s    r�   Úmtp_mlp_layer_typesz%InklingTextConfig.mtp_mlp_layer_types²   s$   € à×ÑÐ*Ø�9˜t×2Ñ2Ñ2Ð2Øó    )<Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚbase_config_keyÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_maprH   ÚintÚ__annotations__rI   rK   rM   rO   rQ   rS   rT   rV   rW   r?   rX   rZ   r[   r\   Úfloatr]   Úlistr^   ÚstrrB   r`   rA   rb   rc   re   rf   r@   ri   rj   rk   Úboolrl   r>   rm   ro   rq   rr   rt   ru   rv   rw   rx   ry   rˆ   Úpropertyr�   r‘   Ú__classcell__©rŒ   s   @r�   r*   r*   <   s®  ø… à€JØ#€Oà+Ø-6Ø*3Ø 0Ø1:Ø/8Ø1:Ø'3Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð !>Ø/Ø/ð 0à5ñ	€Mð €J�Óà&*Ð˜˜t™Ó*Ø€K�ÓØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€HˆcÓØ#%Ð˜SÓ%Ø#%Ð˜SÓ%Ø€L�#ÓØ"Ð˜Ó"Ø€Eˆ3ƒOØ€J�ÓØ&*Ð˜˜t™Ó*Ø"Ð�uÓ"Ø(,€O�T˜#‘Y Ñ%Ó,Ø$(€K��c‘˜TÑ!Ó(Ø#)Ð˜SÓ)Ø€L�%ÓØÐ�cÓØ(,€O�T˜#‘Y Ñ%Ó,Ø"Ð�sÓ"Ø€J�Óà!%Ð˜3Ó%ØÐ�cÓØ Ð˜Ó ØÐ�cÓØ#Ð˜Ó#Ø€K�Óà)-Ð Ó-Ø#'Ð˜5Ó'Ø"Ð�uÓ"Ø#Ð�uÓ#Ø#€L�#˜‘*Ó#Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó à!%€N�C˜$‘JÓ%Ø#(Ð˜DÓ(Ø$(Ð˜TÓ(Ø,0Ð˜˜c™ TÑ)Ó0ô(ð* ñó ðð ñó ôr’   r*   c                   óh   — e Zd ZU dZdZddddœZdZeed<   dZ	eed<   d	Z
eed<   d
Zeed<   dZeed<   y)ÚInklingAudioConfigÚinkling_audioÚaudio_configÚ
n_mel_binsÚmel_vocab_sizeÚtext_hidden_size)Únum_codebooksÚcodebook_sizerK   éP   rg   rJ   r_   r`   rp   rq   N)r“   r”   r•   r–   r—   r›   r©   rœ   r�   rª   r«   r`   rž   rq   r�   r’   r�   r¦   r¦   ¹   sR   … à €JØ$€Oà%Ø)Ø)ñ€Mð €J�ÓØ€N�CÓØ Ð�cÓ Ø€L�%ÓØ#Ð�uÔ#r’   r¦   c                   óœ   — e Zd ZU dZdZddiZdZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)ÚInklingVisionConfigÚinkling_visionÚvision_configrM   Ún_layersrJ   r«   é(   Ú
patch_sizer   Útemporal_patch_sizer   Únum_channelsrY   rK   é   rU   rO   r_   r`   rp   rq   N)r“   r”   r•   r–   r—   r›   r«   rœ   r�   rµ   r¶   r·   rK   rM   rO   r`   rž   rq   r�   r’   r�   r°   r°   Ê   su   … à!€JØ%€OØ(¨*Ð5€Mà Ð�cÓ Ø€J�ÓØ Ð˜Ó Ø€L�#ÓØ€K�ÓØÐ�sÓØ!Ð˜Ó!Ø€L�%ÓØ#Ð�uÔ#r’   r°   c                   óº   ‡ — e Zd ZU dZdZeeedœZdZ	ee
z  dz  ed<   dZee
z  dz  ed<   dZee
z  dz  ed<   dZeed	<   d
Zeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚInklingConfigzETop-level multimodal config (`InklingMMConfig` in the SGLang source).Úinkling_mm_model)r,   r¨   r²   Nr,   r¨   r²   iv Úimage_token_idiu Úaudio_token_idiE Úimage_bos_token_idiT Úaudio_bos_token_idc                 ór  •— |j                  d«      xs i }t        | j                  t        «      r‚| j                  j	                  d|j                  d«      «       | j                  j	                  d|j                  dd«      «       | j                  j	                  d|j                  d«      «       t        | j
                  t        «      r% | j                  d   di | j
                  ¤Ž| _        n%| j
                  € | j                  d   «       | _        t        | j                  t        «      r% | j                  d	   di | j                  ¤Ž| _        n%| j                  € | j                  d	   «       | _        t        | j                  t        «      r% | j                  d
   di | j                  ¤Ž| _        n%| j                  € | j                  d
   «       | _        | j                  j                  | j                  _	        | j                  j                  | j
                  _	        t        ‰| �,  di |¤Ž y )NÚ
mtp_configrv   Únum_nextn_predict_layersrw   Fry   r]   r¨   r²   r,   r�   )r…   Ú
isinstancer,   ÚdictÚ
setdefaultr¨   Úsub_configsr²   rK   r«   r‡   rˆ   )r‰   rŠ   rÁ   rŒ   s      €r�   rˆ   zInklingConfig.__post_init__î   sÕ  ø€ à—Z‘Z Ó-Ò3°ˆ
Ü�d×&Ñ&¬Ô-Ø×Ñ×'Ñ'Ð(8¸*¿.¹.ÐIcÓ:dÔeØ×Ñ×'Ñ'Ð(@À*Ç.Á.ÐQiÐkpÓBqÔrØ×Ñ×'Ñ'Ð(=¸z¿~¹~ÐN_Ó?`Ôaä�d×'Ñ'¬Ô.Ø @ × 0Ñ 0°Ñ @Ñ UÀ4×CTÑCTÑ UˆDÕØ×ÑÐ&Ø @ × 0Ñ 0°Ñ @Ó BˆDÔä�d×(Ñ(¬$Ô/Ø!B ×!1Ñ!1°/Ñ!BÑ!XÀT×EWÑEWÑ!XˆDÕØ×ÑÐ'Ø!B ×!1Ñ!1°/Ñ!BÓ!DˆDÔä�d×&Ñ&¬Ô-Ø>˜t×/Ñ/°Ñ>ÑRÀ×AQÑAQÑRˆDÕØ×ÑÐ%Ø>˜t×/Ñ/°Ñ>Ó@ˆDÔà.2×.>Ñ.>×.JÑ.Jˆ×ÑÔ+Ø-1×-=Ñ-=×-IÑ-Iˆ×ÑÔ*Ü‰ÑÑ' Ó'r’   )r“   r”   r•   Ú__doc__r–   r*   r¦   r°   rÆ   r,   rÄ   r�   r¨   r²   r¼   rœ   r½   r¾   r¿   rˆ   r£   r¤   s   @r�   rº   rº   Û   s�   ø… áOà#€Jà(Ø*Ø,ñ€Kð 48€KÐ" TÑ)¨DÑ0Ó7Ø59€LÐ$ tÑ+¨dÑ2Ó9Ø7;€MÐ&¨Ñ-°Ñ4Ó;Ø €N�CÓ Ø €N�CÓ Ø$Ð˜Ó$Ø$Ð˜Ó$÷(ð (r’   rº   c                   ó   — e Zd Zy)ÚInklingModelOutputWithPastN©r“   r”   r•   r�   r’   r�   rÉ   rÉ   
  ó   „ Ør’   rÉ   c                   ó   — e Zd Zy)ÚInklingCausalLMOutputWithPastNrÊ   r�   r’   r�   rÍ   rÍ     rË   r’   rÍ   c                   ó   — e Zd Zy)ÚInklingRMSNormNrÊ   r�   r’   r�   rÏ   rÏ     rË   r’   rÏ   c                   ó”   ‡ — e Zd ZdZdedefˆ fd„Zdej                  dej                  dej                  dej                  fd	„Zˆ xZ	S )
ÚInklingRelativeLogitsa|  hidden states conditioned relative position bias. `proj` is a trained bank of bias-vs-distance profiles; each token's
    `relative_states` mixes them into one bias value per backward distance
    (`sglang RelLogitsProj` + the FA4 `score_mod`, materialized densely). The bias is zero
    outside `0 <= distance < rel_extent`; causality and padding stay in the attention mask.
    rX   rZ   c                 óŒ   •— t         ‰| �  «        || _        t        j                  t        j                  ||«      «      | _        y ©N)r‡   Ú__init__rZ   ÚnnÚ	ParameterÚtorchÚemptyÚproj)r‰   rX   rZ   rŒ   s      €r�   rÔ   zInklingRelativeLogits.__init__  s0   ø€ Ü‰ÑÔØ$ˆŒÜ—L‘L¤§¡¨U°JÓ!?Ó@ˆ�	r’   Úrelative_statesÚquery_positionsÚkey_positionsÚreturnc                 ój  — || j                   z  j                  dd«      }|d d …d f   |d d d …f   z
  d d d d …d d …f   } |j                  d| j                  dz
  «      j                  g |j
                  d d ¢d‘d‘­Ž }|j                  d|«      }|j                  |dk  || j                  k\  z  d«      S )Nrs   r   r   éÿÿÿÿrn   )rÙ   Ú	transposeÚclamprZ   ÚexpandÚshapeÚgatherÚmasked_fill)r‰   rÚ   rÛ   rÜ   Ú
rel_logitsÚdistanceÚgather_indexÚposition_biass           r�   ÚforwardzInklingRelativeLogits.forward"  sÇ   € ð &¨¯	©	Ñ1×<Ñ<¸QÀÓBˆ
Ø#¢A t GÑ,¨}¸TÂ1¸WÑ/EÑEÀtÈTÒSTÒVWÐGWÑXˆØD�x—~‘~ a¨¯©¸1Ñ)<Ó=×DÑDÐcÀj×FVÑFVÐWYÐXYÐFZÐcÐ\^ÐcÐ`bÒcˆØ"×)Ñ)¨"¨lÓ;ˆØ×(Ñ(¨(°Q©,¸8ÀtÇÁÑ;VÑ)WÐY\Ó]Ð]r’   )
r“   r”   r•   rÇ   rœ   rÔ   r×   ÚTensorrê   r£   r¤   s   @r�   rÑ   rÑ     s^   ø„ ñðA˜cð A¨sõ Að
^àŸ™ð^ð Ÿ™ð^ð —|‘|ð	^ð
 
�‰÷^r’   rÑ   ÚmoduleÚqueryÚkeyÚvaluer9   ÚscalingÚdropoutré   c                 ó&  — t        || j                  «      }	t        || j                  «      }
t        j                  ||	j	                  dd«      «      |z  }|�||z   }|�||z   }t
        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t
        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j	                  dd«      j                  «       }||fS )Nr   r   rß   )ÚdimÚdtype)ÚpÚtrainingrs   )r&   Únum_key_value_groupsr×   Úmatmulrà   rÕ   Ú
functionalÚsoftmaxÚfloat32Útorô   rñ   rö   Ú
contiguous)rì   rí   rî   rï   r9   rð   rñ   ré   rŠ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r�   Úeager_attention_forwardr  0  sð   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ Ø# mÑ3ˆØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r’   c                   óÚ   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  dedz  d	e	e
   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚInklingAttentionÚconfigÚ	layer_idxc                 ó¼  •— t         ‰| �  «        || _        || _        |j                  | j                     dk(  | _        | j
                  r|j                  n|j                  | _        | j
                  r|j                  n|j                  | _
        | j
                  r|j                  n|j                  | _        | j                  | j                  z  | _        | j
                  r|j                  nd | _        | j
                  r|j                  n|j                   | _        d| j                  z  | _        |j$                  | _        d| _        t)        j*                  |j,                  | j                  | j                  z  d¬«      | _        t)        j*                  |j,                  | j                  | j                  z  d¬«      | _        t)        j*                  |j,                  | j                  | j                  z  d¬«      | _        t)        j*                  |j,                  | j                  |j4                  z  d¬«      | _        t)        j*                  | j                  | j                  z  |j,                  d¬«      | _        t;        | j                  | j                  z  |j<                  |d¬«      | _        t;        | j                  | j                  z  |j<                  |d¬«      | _         tC        | j                  |jD                  ¬	«      | _#        tC        | j                  |jD                  ¬	«      | _$        tK        |j4                  | j                   «      | _&        y )
Nr{   ç      ð?TF©Úbiasr   )Úconv_idxrs   ©Úeps)'r‡   rÔ   r  r  r^   Ú
is_slidingrW   rS   rT   rO   Ú	num_headsrV   rQ   r÷   r?   rD   rZ   rð   ro   Ú	is_causalrÕ   ÚLinearrK   Úq_projÚk_projÚv_projrX   Úr_projÚo_projÚInklingShortConvolutionrF   Úk_sconvÚv_sconvrÏ   r`   Úq_normÚk_normrÑ   Úrel_logits_proj©r‰   r  r  rŒ   s      €r�   rÔ   zInklingAttention.__init__M  sg  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ ×,Ñ,¨T¯^©^Ñ<Ð@PÑPˆŒØ/3¯ª˜×+Ò+ÀFÇOÁOˆŒØ;?¿?º?˜×7Ò7ÐPV×PjÑPjˆŒØEIÇ_Â_ 6×#AÒ#AÐZ`×ZtÑZtˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø<@¿OºO˜f×8Ò8ÐQUˆÔØ8<¿º˜&×4Ò4ÈV×M^ÑM^ˆŒà˜TŸ]™]Ñ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘i × 2Ñ 2°D·N±NÀTÇ]Á]Ñ4RÐY^Ô_ˆŒÜ—i‘i × 2Ñ 2°D×4LÑ4LÈtÏ}É}Ñ4\ÐchÔiˆŒÜ—i‘i × 2Ñ 2°D×4LÑ4LÈtÏ}É}Ñ4\ÐchÔiˆŒÜ—i‘i × 2Ñ 2°D·N±NÀVÇ\Á\Ñ4QÐX]Ô^ˆŒÜ—i‘i §¡°·±Ñ >À×@RÑ@RÐY^Ô_ˆŒÜ.Ø×$Ñ$ t§}¡}Ñ4°f×6NÑ6NÐPYÐdeô
ˆŒô /Ø×$Ñ$ t§}¡}Ñ4°f×6NÑ6NÐPYÐdeô
ˆŒô % T§]¡]¸×8KÑ8KÔLˆŒÜ$ T§]¡]¸×8KÑ8KÔLˆŒÜ4°V·\±\À4Ç?Á?ÓSˆÕr’   Nr8   r9   Ú	conv_maskÚpast_key_valuesrŠ   rÝ   c                 óH  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }| j                  | j	                  |«      ||¬«      }	| j                  | j                  |«      ||¬«      }
| j                  |«      }| j                  |j                  |«      «      j                  dd«      }| j                  |	j                  |«      «      j                  dd«      }	|
j                  |«      j                  dd«      }
|j                   d   }|�[|j                  || j                  «      \  }}|j                  | j                  «      }|j                  |	|
| j                  «      \  }	}
n|	j                   d   }d\  }}t!        j"                  ||j$                  ¬«      |z   }t!        j"                  ||j$                  ¬«      |z   } |j                  g |¢| j&                  ‘d‘­Ž }| j)                  |||«      }| j*                  së| j,                  j.                  �Õ|dz   j1                  «       }d| j,                  j2                  t!        j4                  || j,                  j.                  z  j7                  d¬«      «      z  z   }|j                  dddd«      }|j1                  «       |z  j9                  |j:                  «      }|j1                  «       |z  j9                  |j:                  «      }t=        j>                  | j,                  j@                  tB        «      } || ||	|
|f| jD                  sd	n| jF                  | jH                  | jJ                  |d
œ|¤Ž\  }} |jL                  g |¢d‘­Ž jO                  «       }| jQ                  |«      }||fS )Nrß   ©r  r  rs   r   )r   r   ©Údevicer  )Úminrn   )rñ   rð   rD   ré   ))rã   rS   r  r  r  r  r  r  r  Úviewrà   r  Úget_mask_sizesr  Úget_query_offsetÚupdater×   Úaranger#  r  r  r  r  r[   rž   r\   Úlográ   rü   rô   r   Úget_interfaceÚ_attn_implementationr  rö   ro   rð   rD   Úreshaperý   r  )r‰   r8   r9   r  r  rŠ   Úinput_shapeÚhidden_shapeÚquery_statesrþ   rÿ   rÚ   Úq_lengthÚ	kv_lengthÚ	kv_offsetÚq_offsetÚkv_positionsÚq_positionsré   Úeffective_nÚtauÚattention_interfacer  r   s                           r�   rê   zInklingAttention.forwardl  s]  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—\‘\ $§+¡+¨mÓ"<ÈoÐir�\Ósˆ
Ø—|‘| D§K¡K°Ó$>ÐP_Ðkt�|ÓuˆØŸ+™+ mÓ4ˆà—{‘{ <×#4Ñ#4°\Ó#BÓC×MÑMÈaÐQRÓSˆØ—[‘[ §¡°Ó!>Ó?×IÑIÈ!ÈQÓOˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà×%Ñ% aÑ(ˆØÐ&à#2×#AÑ#AÀ(ÈDÏNÉNÓ#[Ñ ˆI�yØ&×7Ñ7¸¿¹ÓGˆHà'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ™à"×(Ñ(¨Ñ+ˆIØ"&ÑˆH�iä—|‘| I°m×6JÑ6JÔKÈiÑWˆÜ—l‘l 8°M×4HÑ4HÔIÈHÑTˆØ.˜/×.Ñ.ÐP°ÐP¸T¿^¹^ÐPÈRÒPˆØ×,Ñ,¨_¸kÈ<ÓXˆð �Š 4§;¡;×#BÑ#BÐ#NØ&¨™?×1Ñ1Ó3ˆKØ˜Ÿ™×5Ñ5¼¿	¹	Ø˜tŸ{™{×>Ñ>Ñ>×EÑEÈ#ÐEÓNó9ñ ñ ˆCð —(‘(˜1˜a  QÓ'ˆCØ(×.Ñ.Ó0°3Ñ6×:Ñ:¸<×;MÑ;MÓNˆLØ*×0Ñ0Ó2°SÑ8×<Ñ<¸]×=PÑ=PÓQˆMä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.Ø'ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r’   ©NN)r“   r”   r•   r*   rœ   rÔ   r×   rë   r   r   r   Útuplerê   r£   r¤   s   @r�   r  r  L  s    ø„ ðTÐ0ð T¸Sõ TðF *.Ø(,ñA)à—|‘|ðA)ð Ÿ™ tÑ+ðA)ð —<‘< $Ñ&ð	A)ð
  ™ðA)ð Ð+Ñ,ðA)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷A)r’   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú
InklingMLPr  c                 ó®   •— t         ‰| �  |«       t        |j                     | _        t        j                  t        j                  d«      «      | _	        y ©Nrs   )
r‡   rÔ   r   re   Úact_fnrÕ   rÖ   r×   ÚonesÚglobal_scale©r‰   r  rŒ   s     €r�   rÔ   zInklingMLP.__init__±  s;   ø€ Ü‰Ñ˜Ô Ü˜V×.Ñ.Ñ/ˆŒÜŸL™L¬¯©°A«Ó7ˆÕr’   r8   rÝ   c                 ó¢   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|| j                  z  S rÓ   )Ú	down_projr@  Ú	gate_projÚup_projrB  ©r‰   r8   s     r�   rê   zInklingMLP.forward¶  sE   € ØŸ™ t§{¡{°4·>±>À-Ó3PÓ'QÐTX×T`ÑT`ÐanÓToÑ'oÓpˆØ˜t×0Ñ0Ñ0Ð0r’   )	r“   r”   r•   r*   rÔ   r×   rë   rê   r£   r¤   s   @r�   r=  r=  °  s+   ø„ ð8Ð0õ 8ð
1 U§\¡\ð 1°e·l±l÷ 1r’   r=  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚInklingExpertsr  c                 óh   •— t         ‰| �  |«       |j                  | _        |j                  | _        y rÓ   )r‡   rÔ   r@   Únum_expertsrf   Úintermediate_dimrC  s     €r�   rÔ   zInklingExperts.__init__¼  s,   ø€ Ü‰Ñ˜Ô Ø!×2Ñ2ˆÔØ &× <Ñ <ˆÕr’   )r“   r”   r•   r*   rÔ   r£   r¤   s   @r�   rJ  rJ  »  s   ø„ ð=Ð0÷ =ñ =r’   rJ  c                   ór   ‡ — e Zd Zˆ fd„Zdeej                  ej                  ej                  f   fd„Zˆ xZS )ÚInklingTopkRouterc                 óT  •— t         ‰| �  «        |j                  | _        |j                  | _        | j                  | j                  z   | _        |j                  | _        |j                  | _        |j                  | _
        t        j                  t        j                  | j
                  |j                  «      «      | _        t        j                  t        j                   d«      «      | _        t        j                  t        j                  | j                  «      «      | _        y r?  )r‡   rÔ   r@   rL  rj   Ún_total_expertsrK   Ú
hidden_dimrl   ri   Útop_krÕ   rÖ   r×   rØ   ÚweightrA  rB  Úe_score_correction_biasrC  s     €r�   rÔ   zInklingTopkRouter.__init__Ã  sË   ø€ Ü‰ÑÔØ!×2Ñ2ˆÔØ &× 7Ñ 7ˆÔØ#×/Ñ/°$×2GÑ2GÑGˆÔØ ×,Ñ,ˆŒØ!×-Ñ-ˆÔØ×/Ñ/ˆŒ
ä—l‘l¤5§;¡;¨t×/CÑ/CÀV×EWÑEWÓ#XÓYˆŒÜŸL™L¬¯©°A«Ó7ˆÔÜ')§|¡|´E·K±KÀ×@PÑ@PÓ4QÓ'RˆÕ$r’   rÝ   c                 ó  — |j                  d| j                  «      }t        j                  || j                  «      }|j                  «       }|dd | j                   …f   }|| j                  z   }t        j                  || j                  dd¬«      d   }|dd | j                   …f   }|d| j                   d …f   }	t        j                  |j                  d|«      |	gd¬«      }
t        j                  |
«      }t        j                  |t        j                  |dd¬«      z
  «      }|| j                   z  | j"                  z  }|d| j                   d …f   j%                  «       }|dd | j                  …f   j%                  «       }||||fS )	Nrß   .F)ró   Úsortedrs   ©ró   T)ró   Úkeepdim)r-  rR  ÚFÚlinearrT  Úsigmoidrj   rU  r×   ÚtopkrS  Úcaträ   Ú
logsigmoidÚexpÚ	logsumexprl   rB  rý   )r‰   r8   ÚflatÚrouter_logitsÚscoresÚrouted_scoresÚscores_for_choiceÚtopk_indicesÚrouted_logitsÚshared_logitsÚtopk_logitsÚtopk_log_probsÚtopk_weightsÚshared_gammass                 r�   rê   zInklingTopkRouter.forwardÐ  s…  € Ø×$Ñ$ R¨¯©Ó9ˆÜŸ™  t§{¡{Ó3ˆð ×&Ñ&Ó(ˆØ˜sÐ$< t×'<Ñ'<Ð&<Ð$<Ð<Ñ=ˆØ)¨D×,HÑ,HÑHÐÜ—z‘zÐ"3°T·Z±ZÀRÐPUÔVÐWXÑYˆà% cÐ+C¨d×.CÑ.CÐ-CÐ+CÐ&CÑDˆØ% c¨D×,AÑ,AÐ+AÑ+CÐ&CÑDˆÜ—i‘i ×!5Ñ!5°b¸,Ó!GÈÐ WÐ]_Ô`ˆÜŸ™ kÓ2ˆÜ—y‘y ´%·/±/À.ÐVXÐbfÔ2gÑ!gÓhˆà# d×&6Ñ&6Ñ6¸×9JÑ9JÑJˆà$ S¨4×+@Ñ+@Ð*@Ñ*BÐ%BÑC×NÑNÓPˆØ# C¨¨4¯:©:¨Ð$5Ñ6×AÑAÓCˆà˜l¨L¸-ÐGÐGr’   )	r“   r”   r•   rÔ   r;  r×   rë   rê   r£   r¤   s   @r�   rO  rO  Â  s/   ø„ ôSðH¨¨e¯l©l¸E¿L¹LÈ%Ï,É,Ð.VÑ(W÷ Hr’   rO  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚInklingSharedExpertsc                 ó  •— t         ‰| �  «        |j                  | _        |j                  }t	        j
                  t        j                  |j                  ||j                  «      «      | _	        t	        j
                  t        j                  |j                  ||j                  «      «      | _
        t	        j
                  t        j                  |j                  |j                  |«      «      | _        t        |j                     | _        y rÓ   )r‡   rÔ   rj   rf   rÕ   rÖ   r×   rØ   rK   rF  rG  rE  r   re   r@  )r‰   r  rM  rŒ   s      €r�   rÔ   zInklingSharedExperts.__init__é  s¼   ø€ Ü‰ÑÔØ &× 7Ñ 7ˆÔØ!×7Ñ7Ðô
 Ÿ™¤e§k¡k°&×2IÑ2IÐK[Ð]c×]oÑ]oÓ&pÓqˆŒÜ—|‘|¤E§K¡K°×0GÑ0GÐIYÐ[a×[mÑ[mÓ$nÓoˆŒÜŸ™¤e§k¡k°&×2IÑ2IÈ6×K]ÑK]Ð_oÓ&pÓqˆŒÜ˜V×.Ñ.Ñ/ˆ�r’   c                 ó¸  — |j                   }|j                  dd|d   «      j                  | j                  dd«      }|j                  d| j                  d«      j	                  dd«      }t        j                  || j                  j	                  dd«      «      }t        j                  || j                  j	                  dd«      «      }| j                  |«      |z  |z  }t        j                  || j                  j	                  dd«      «      }|j                  «       j                  d¬«      j                  |j                  «      }|j                  |«      S )Nrs   rß   r   r   rX  )rã   r-  râ   rj   rà   r×   ÚbmmrF  rG  r@  rE  rž   Úsumrü   rô   r%  )	r‰   r8   Úgammasr.  ÚgateÚupÚ	activatedÚdownÚouts	            r�   rê   zInklingSharedExperts.forwardö  s  € Ø#×)Ñ)ˆØ%×-Ñ-¨a°°[À±_ÓE×LÑLÈT×MbÑMbÐdfÐhjÓkˆØ—‘  D×$9Ñ$9¸1Ó=×GÑGÈÈ1ÓMˆä�y‰y˜¨¯©×(@Ñ(@ÀÀAÓ(FÓGˆÜ�Y‰Y�} d§l¡l×&<Ñ&<¸QÀÓ&BÓCˆØ—K‘K Ó%¨Ñ*¨VÑ3ˆ	Ü�y‰y˜ D§N¡N×$<Ñ$<¸QÀÓ$BÓCˆà�j‰j‹l×Ñ 1ÐÓ%×(Ñ(¨×)<Ñ)<Ó=ˆØ�x‰x˜Ó$Ð$r’   )r“   r”   r•   rÔ   rê   r£   r¤   s   @r�   ro  ro  è  s   ø„ ô0ö%r’   ro  c                   óB   ‡ — e Zd ZdZˆ fd„Zdej                  fd„Zˆ xZS )Ú
InklingMoEz7Gate -> routed experts (+ shared experts), TML flavour.c                 ó�   •— t         ‰| �  «        || _        t        |«      | _        t        |«      | _        t        |«      | _        y rÓ   )	r‡   rÔ   r  rO  ru  rJ  Úexpertsro  Úshared_expertsrC  s     €r�   rÔ   zInklingMoE.__init__  s:   ø€ Ü‰ÑÔØˆŒÜ% fÓ-ˆŒ	Ü% fÓ-ˆŒÜ2°6Ó:ˆÕr’   rÝ   c                 óø   — |}|j                   }| j                  |«      \  }}}}|j                  d|j                   d   «      } | j                  |||«      j                  |Ž }|| j	                  ||¬«      z   }|S )Nrß   )rt  )rã   ru  r%  r}  r~  )r‰   r8   Ú	residualsr.  Ú_rl  rg  rm  s           r�   rê   zInklingMoE.forward  s†   € Ø!ˆ	Ø#×)Ñ)ˆØ7;·y±yÀÓ7OÑ4ˆˆ<˜ }Ø%×*Ñ*¨2¨}×/BÑ/BÀ2Ñ/FÓGˆØT˜Ÿ™ ]°LÀ,ÓO×TÑTÐVaÐbˆØ%¨×(;Ñ(;¸IÈmÐ(;Ó(\Ñ\ˆØÐr’   )	r“   r”   r•   rÇ   rÔ   r×   rë   rê   r£   r¤   s   @r�   r{  r{    s   ø„ ÙAô;ð¨¯©÷ r’   r{  Úcausal_conv1d_updater8   Ú
conv_staterT  r
  Ú
activationc                 ó¦  — | j                   \  }}}|j                   d   }t        j                  || gd¬«      j                  |j                  «      }	|j                  |	d d …d d …| d …f   «       t        j                  |	|j                  d«      |d|¬«      }
|
d d …d d …| d …f   }
|�t        |   |
«      }
|
j                  | j                  «      S )Nrß   rX  rs   r   )ÚpaddingÚgroups)
rã   r×   r^  rü   rô   Úcopy_rZ  Úconv1dÚ	unsqueezer   )r8   rƒ  rT  r
  r„  r�  rK   Úseq_lenÚ	state_lenÚhidden_states_newry  s              r�   r‚  r‚    sË   € ð ,×1Ñ1Ñ€A€{�GØ× Ñ  Ñ$€IäŸ	™	 :¨}Ð"=À2ÔF×IÑIÈ&Ï,É,ÓWÐØ×ÑÐ&¢qª!¨i¨Z©[Ð'8Ñ9Ô:Ü
�(‰(Ð$ f×&6Ñ&6°qÓ&9¸4ÈÐS^Ô
_€CØ
Ša’�W�H‘IˆoÑ
€CØÐÜ�ZÑ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r’   Úcausal_conv1d_fnc                 ó8  — | j                   \  }}}|j                   d   dz
  }t        j                  | j                  |j                  «      |j                  d«      |||¬«      d d …d d …d |…f   }	|�t        |   |	«      }	|	j                  | j                  «      S )Nrß   rs   )rT  r
  r†  r‡  )rã   rZ  r‰  rü   rô   rŠ  r   )
r8   rT  r
  r„  rŠ   r�  rK   r‹  r†  ry  s
             r�   rŽ  rŽ  ,  s    € ð ,×1Ñ1Ñ€A€{�GØ�l‰l˜2Ñ Ñ"€Gä
�(‰(Ø×Ñ˜Ÿ™Ó&Ø×Ñ Ó"ØØØôò ŠˆHˆWˆH€nñ€Cð ÐÜ�ZÑ  Ó%ˆØ�6‰6�-×%Ñ%Ó&Ð&r’   c                   óž   ‡ — e Zd Zdedededefˆ fd„Z ed«      	 	 ddej                  d	edz  d
ej                  dz  de	e
   fd„«       Zˆ xZS )r  rK   rA   r  r  c                 ó’   •— t         ‰| �  «        || _        || _        || _        t        j                  |||||dz
  d¬«      | _        y )Nrs   F)Úin_channelsÚout_channelsÚkernel_sizer‡  r†  r
  )r‡   rÔ   r  r  rA   rÕ   ÚConv1dr‰  )r‰   rK   rA   r  r  rŒ   s        €r�   rÔ   z InklingShortConvolution.__init__E  sK   ø€ Ü‰ÑÔØ"ˆŒØ ˆŒØ 0ˆÔä—i‘iØ#Ø$Ø(ØØ$ qÑ(Øô
ˆ�r’   r‰  Nr8   r  r  rŠ   c                 ó¸  — |j                   }|j                  «       }|}t        ||«      }|j                  d   }|j	                  dd«      }|d uxr& |j                  | j                  | j                  «      }|rž|dk(  r™|j                  | j                     j                  sv|j                  | j                     j                  | j                     }	t        ||	| j                  j                  j                  d«      | j                  j                  «      }n›|�3|j!                  || j                  | j                  | j"                  ¬«      }t%        || j                  j                  j                  d«      | j                  j                  |j'                  d«      ¬«      }|r|d d …d d …| d …f   }|j	                  dd«      }||z   j)                  |¬«      }|S )Nrs   r   )Ú	state_idxrA   Úseq_idx)r˜  )rô   )rô   rž   r(   rã   rà   Úhas_previous_stater  r  r:   Úrecord_pastÚconv_statesr‚  r‰  rT  Úsqueezer
  Úupdate_conv_staterA   rŽ  r…   rü   )
r‰   r8   r  r  rŠ   Úinput_dtypeÚresidualr‹  Úuse_precomputed_statesrƒ  s
             r�   rê   zInklingShortConvolution.forwardT  sµ  € ð $×)Ñ)ˆØ%×+Ñ+Ó-ˆà ˆÜ4°]ÀIÓNˆØ×%Ñ% aÑ(ˆØ%×/Ñ/°°1Ó5ˆà!0¸Ð!<ò "
À×AcÑAcØ�N‰N˜DŸM™MóB
Ðñ " g°¢l¸?×;QÑ;QÐRV×R`ÑR`Ñ;a×;mÒ;mØ(×/Ñ/°·±Ñ?×KÑKÈDÏMÉMÑZˆJä0Ø˜z¨4¯;©;×+=Ñ+=×+EÑ+EÀaÓ+HÈ$Ï+É+×JZÑJZó‰Mð Ð*Ø /× AÑ AØ! 4§>¡>¸T¿]¹]Ð]a×]rÑ]rð !Bó !�ô -Ø˜tŸ{™{×1Ñ1×9Ñ9¸!Ó<¸d¿k¹k×>NÑ>NÐX^×XbÑXbÐclÓXmôˆMñ
 &Ø -ªa²°W°H±I¨oÑ >�à%×/Ñ/°°1Ó5ˆØ&¨Ñ1×5Ñ5¸KÐ5ÓHˆØÐr’   r:  )r“   r”   r•   rœ   rÔ   r   r×   rë   r   r   r   rê   r£   r¤   s   @r�   r  r  C  sƒ   ø„ ð
 Cð 
¸3ð 
È3ð 
ÐZ]õ 
ñ ˜HÓ%ð )-Ø)-ñ	*à—|‘|ð*ð  ™ð*ð —<‘< $Ñ&ð	*ð
 Ð+Ñ,ò*ó &ô*r’   r  c                   ó¸   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  dedz  d	e	e
   d
ej                  fd„Zˆ xZS )ÚInklingDecoderLayerr  r  c                 ó(  •— t         ‰| �  «        |j                  | _        t        ||«      | _        |j
                  |   dk(  rt        |«      | _        nt        |«      | _        t        |j                  |j                  «      | _        t        |j                  |j                  «      | _        |j                  |   | _        t        |j                  |j                   |d¬«      | _        t        |j                  |j                   |d¬«      | _        y )Nr   r   )r  r  r   )r‡   rÔ   rK   r  Ú	self_attnrb   r{  Úmlpr=  rÏ   r`   Úinput_layernormÚpost_attention_layernormr^   Ú
layer_typer  rA   Ú
attn_sconvÚ	mlp_sconvr  s      €r�   rÔ   zInklingDecoderLayer.__init__ƒ  sã   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ)¨&°)Ó<ˆŒà×!Ñ! )Ñ,°Ò8Ü! &Ó)ˆD�Hä! &Ó)ˆDŒHä-¨f×.@Ñ.@À&×BUÑBUÓVˆÔÜ(6°v×7IÑ7IÈ6×K^ÑK^Ó(_ˆÔ%Ø ×,Ñ,¨YÑ7ˆŒÜ1Ø×Ñ × 7Ñ 7À9ÐWXô
ˆŒô 1Ø×Ñ × 7Ñ 7À9ÐWXô
ˆ�r’   Nr8   r9   r  r  rŠ   rÝ   c                 ó  — |}| j                  |«      } | j                  d||||dœ|¤Ž\  }}| j                  |||¬«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |||¬«      }||z   }|S )N)r8   r9   r  r  r!  r�   )r¦  r¤  r©  r§  r¥  rª  )r‰   r8   r9   r  r  rŠ   rŸ  r�  s           r�   rê   zInklingDecoderLayer.forward—  s²   € ð !ˆØ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø)ØØ+ñ	
ð
 ñ
Ñˆ�qð Ÿ™¨ÀÐbk˜ÓlˆØ  =Ñ0ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØŸ™ }ÀoÐaj˜ÓkˆØ  =Ñ0ˆØÐr’   )NNN)r“   r”   r•   r*   rœ   rÔ   r×   rë   r   r   r   rê   r£   r¤   s   @r�   r¢  r¢  ‚  s…   ø„ ð
Ð0ð 
¸Sõ 
ð. /3Ø)-Ø(,ñà—|‘|ðð Ÿ™ tÑ+ðð —<‘< $Ñ&ð	ð
  ™ðð Ð+Ñ,ðð 
�‰÷r’   r¢  c                   óŠ   ‡ — e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdgZg d¢ZeedœZ ej(                  «       ˆ fd	„«       Zˆ xZS )
ÚInklingPreTrainedModelÚmodelTr¢  r  Fzmodel\.mtp\..*)r©  rª  r  r  )r8   Ú
attentionsc                 ó¤  •— t         ‰| �  |«       | j                  j                  «       j                  }t        |t        «      r#t        j                  |j                  d|¬«       y t        |t        «      r t        j                  |j                  «       y t        |t        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       y t        |t         «      rat        j                  |j"                  d|¬«       t        j                  |j                  «       t        j$                  |j&                  «       y t        |t(        «      rgt        j                  |j*                  d|¬«       t        j                  |j,                  d|¬«       t        j                  |j                  d|¬«       y t        |t.        «      rlt1        | j                  d| j                  «      }t        j2                  |j4                  t7        j8                  |j:                  «      |j<                  z  «       y y )Nrn   )ÚmeanÚstdr¨   )r‡   Ú_init_weightsr  Úget_text_configrq   rÃ   rÑ   ÚinitÚnormal_rÙ   r=  Úones_rB  rJ  Úgate_up_projrE  rO  rT  Úzeros_rU  ro  rF  rG  ÚInklingAudioModelEmbeddingsÚgetattrrˆ  Úaudio_tokens_offsetsr×   r)  r©   rª   )r‰   rì   r²  r¨   rŒ   s       €r�   r³  z$InklingPreTrainedModel._init_weightsÈ  s�  ø€ ä‰Ñ˜fÔ%Ø�k‰k×)Ñ)Ó+×=Ñ=ˆÜ�fÔ3Ô4Ü�L‰L˜Ÿ™¨3°CÖ8Ü˜¤
Ô+Ü�J‰J�v×*Ñ*Õ+Ü˜¤Ô/Ü�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 1Ô2Ü�L‰L˜Ÿ™¨S°cÕ:Ü�J‰J�v×*Ñ*Ô+Ü�K‰K˜×6Ñ6Õ7Ü˜Ô 4Ô5Ü�L‰L˜×)Ñ)°¸Õ=Ü�L‰L˜Ÿ™¨c°sÕ;Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô ;Ô<ô # 4§;¡;°ÀÇÁÓLˆLÜ�J‰JØ×+Ñ+Ü—‘˜\×4Ñ4Ó5¸×8SÑ8SÑSõð	 =r’   )r“   r”   r•   rº   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ"_keys_to_ignore_on_load_unexpectedÚ_keep_in_fp32_modules_strictr¢  r  Ú_can_record_outputsr×   Úno_gradr³  r£   r¤   s   @r�   r­  r­  ³  s~   ø„ à €LØÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5Ðð !ÐØ€NØÐØ"ÐØ"'ÐØ*;Ð)<Ð&Ú#TÐ à,Ø&ñÐð
 €U‡]�]ƒ_óó ôr’   r­  c                   óþ   ‡ — e Zd ZU eed<   defˆ fd„Zeee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚInklingTextModelr  c           	      ó6  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        d| _        | j'                  «        y c c}w )Nr  F)r‡   rÔ   rr   Úpadding_idxrH   rÕ   Ú	EmbeddingrK   r2   Ú
ModuleListrƒ   rM   r¢  r:   rÏ   r`   r;   Ú
embed_normÚgradient_checkpointingÚ	post_initr  s      €r�   rÔ   zInklingTextModel.__init__é  sÔ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü(¨×);Ñ);À×ATÑATÔUˆŒØ&+ˆÔ#ð 	�‰Õùò fs   ÂDNr6   r9   Úposition_idsr  r7   Ú	use_cacherŠ   rÝ   c                 ó¼  — |d u |d uz  rt        d«      ‚|€ | j                  | j                  |«      «      }|r|€t        | j                  ¬«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s3| j                  ||||dœ}
t        di |
¤Žt        di |
¤Žt        di |
¤Ždœ}	|}t!        | j"                  «      D ]8  \  }}| j                  j$                  |   dk(  rd	nd
} ||f|	|   |	d   |dœ|¤Ž}Œ: | j'                  |«      }t)        ||¬«      S )Nú:You must specify exactly one of input_ids or inputs_embeds)r  r   rs   r"  ©r  r7   r9   r  rÔ  ©Úfull_attentionÚsliding_attentionÚlinear_attentionr|   rÚ  rÛ  rÜ  )r9   r  r  )Úlast_hidden_stater  r�   )Ú
ValueErrorrÑ  r2   r	   r  Úget_seq_lengthr×   r)  rã   r#  rŠ  rÃ   rÄ   r   r   r   Ú	enumerater:   r^   r;   r   )r‰   r6   r9   rÔ  r  r7   rÕ  rŠ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr8   r‹   Údecoder_layerÚattention_types                  r�   rê   zInklingTextModel.forwardù  s˜  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ŸO™O¨D×,=Ñ,=¸iÓ,HÓIˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%UÜ$CÑ$RÀkÑ$Rñ#Ðð &ˆÜ )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ø15·±×1HÑ1HÈÑ1KÈxÒ1WÑ-Ð]pˆNÙ)Øðà2°>ÑBØ-Ð.@ÑAØ /ñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r’   )NNNNNN)r“   r”   r•   r*   r�   rÔ   r   r   r   r×   Ú
LongTensorrë   r   ÚFloatTensorr¡   r   r   r   rê   r£   r¤   s   @r�   rÌ  rÌ  å  sÔ   ø… àÓðÐ0õ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð ˜$‘;ð6
ð Ð+Ñ,ð6
ð 
!ò6
ó ó ó  ô6
r’   rÌ  c                   ó  — e Zd Zi ZddiZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
edz  deej                  z  dee   defd„«       «       Zy)ÚInklingForCausalLMÚlm_headÚrowwise_split_inputNr6   r9   rÔ  r  r7   ÚlabelsrÕ  Úlogits_to_keeprŠ   rÝ   c	           
      óþ  —  | j                   d||||||dœ|	¤Ž}
|
j                  | j                  j                  z  }t	        |t
        «      rt        | d «      n|}| j                  |d d …|d d …f   «      }| j                  j                  }|�||j                  d   k  r	|dd |…f   }d }|�# | j                  d|||j                  d   dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )N)r6   r9   rÔ  r  r7   rÕ  rß   .©Úlogitsrì  rH   )Úlossrð  r  r8   r¯  r�   )r®  rÝ  r  r>   rÃ   rœ   Úslicerê  rI   rã   Úloss_functionrÍ   r  r8   r¯  )r‰   r6   r9   rÔ  r  r7   rì  rÕ  rí  rŠ   Úoutputsr8   Úslice_indicesrð  rI   rñ  s                   r�   rê   zInklingForCausalLM.forward:  s&  € ð �$—*‘*ð 
ØØ)Ø%Ø+Ø'Øñ
ð ñ
ˆð  ×1Ñ1°D·K±K×4[Ñ4[Ñ[ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ"Ÿk™k×=Ñ=ÐØÐ*Ð/BÀVÇ\Á\ÐRTÑEUÒ/UØ˜CÐ!5Ð"5Ð!5Ð5Ñ6ˆFàˆØÐØ%�4×%Ñ%Ðj¨V¸FÈvÏ|É|Ð\^ÑO_ÑjÐciÑjˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r’   )NNNNNNNr   )r“   r”   r•   Ú_tied_weights_keysÚ_tp_planr   r   r×   ræ  rë   r   rç  r¡   rœ   r   r   rÍ   rê   r�   r’   r�   ré  ré  5  sõ   „ àÐØÐ0Ð1€HàØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ(
à×#Ñ# dÑ*ð(
ð Ÿ™ tÑ+ð(
ð ×&Ñ&¨Ñ-ð	(
ð
  ™ð(
ð ×(Ñ(¨4Ñ/ð(
ð × Ñ  4Ñ'ð(
ð ˜$‘;ð(
ð ˜eŸl™lÑ*ð(
ð Ð+Ñ,ð(
ð 
'ò(
ó ó ñ(
r’   ré  c                   ó   — e Zd Zy)rº  NrÊ   r�   r’   r�   rº  rº  g  s   … r’   rº  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚInklingAudioModelr  c                 ó|   •— t         ‰| �  |«       t        |«      | _        t	        |j
                  d¬«      | _        y )Nr_   r  )r‡   rÔ   rº  Úembed_audio_tokensrÏ   r«   r;   rC  s     €r�   rÔ   zInklingAudioModel.__init__k  s1   ø€ Ü‰Ñ˜Ô Ü"=¸fÓ"EˆÔÜ" 6×#:Ñ#:ÀÔEˆ�	r’   Úaudio_input_idsrÝ   c                 ó`   — | j                  |«      }| j                  |«      }t        ||¬«      S )N©rÝ  Úpooler_output)rü  r;   r   )r‰   rý  r8   s      r�   rê   zInklingAudioModel.forwardp  s3   € Ø×/Ñ/°Ó@ˆØŸ	™	 -Ó0ˆÜ)Ø+Ø'ô
ð 	
r’   )	r“   r”   r•   r¦   rÔ   r×   rë   rê   r£   r¤   s   @r�   rú  rú  j  s-   ø„ ðFÐ1õ Fð

 u§|¡|ð 
¸¿¹÷ 
r’   rú  c            
       ó¤   ‡ — e Zd Zdededededef
ˆ fd„Zdej                  dej                  fd	„Zdej                  dej                  fd
„Z	ˆ xZ
S )ÚInklingVisionEncoderLayerÚ	input_dimÚ
output_dimÚt_foldÚhw_foldÚadd_normc                 óª   •— t         ‰| �  «        t        j                  ||d¬«      | _        |rt        |«      | _        || _        || _        || _	        y ©NFr	  )
r‡   rÔ   rÕ   r  Ú
projectionrÏ   Ú
layer_normr  r  r  )r‰   r  r  r  r  r  rŒ   s         €r�   rÔ   z"InklingVisionEncoderLayer.__init__z  sF   ø€ Ü‰ÑÔÜŸ)™) I¨zÀÔFˆŒÙÜ,¨ZÓ8ˆDŒOØˆŒØˆŒØ ˆ�r’   r8   rÝ   c           
      ó˜  — |j                   \  }}}}}|| j                  z  }|| j                  z  }|| j                  z  }	|j                  ||| j                  || j                  |	| j                  |«      }|j	                  dddddddd«      }|j                  ||||	| j                  | j                  z  | j                  z  |z  «      }|S )	z€
        Convert a tensor of shape (B, T, H, W, C) to a tensor of shape (B, T // t, H // hw, W //  hw, C * (t * hw**2))
        r   rs   r   é   r   ra   rh   é   )rã   r  r  r-  Úpermute)
r‰   r8   ÚBÚTÚHÚWÚCÚt_newÚh_newÚw_news
             r�   Úfold_timespace_to_depthz1InklingVisionEncoderLayer.fold_timespace_to_depthƒ  sÐ   € ð &×+Ñ+‰ˆˆ1ˆa��Aà�T—[‘[Ñ ˆØ�T—\‘\Ñ!ˆØ�T—\‘\Ñ!ˆà%×-Ñ-¨a°¸¿¹ÀUÈDÏLÉLÐZ_Ðae×amÑamÐopÓqˆà%×-Ñ-¨a°°A°q¸!¸QÀÀ1ÓEˆØ%×-Ñ-¨a°¸¸uÀdÇkÁkÐTX×T`ÑT`ÑF`Ðcg×coÑcoÑFoÐrsÑFsÓtˆØÐr’   c                 óê   — | j                   dkD  s| j                  dkD  r| j                  |«      }| j                  |«      }| j                  r&| j                  |«      }t        j                  |«      }|S r?  )r  r  r  r
  r  r  rZ  ÚgelurH  s     r�   rê   z!InklingVisionEncoderLayer.forward“  s_   € Ø�<‰<˜!Ò˜tŸ{™{¨QšØ ×8Ñ8¸ÓGˆMàŸ™¨Ó6ˆØ�=Š=Ø ŸO™O¨MÓ:ˆMÜŸF™F =Ó1ˆMØÐr’   )r“   r”   r•   rœ   r¡   rÔ   r×   rë   r  rê   r£   r¤   s   @r�   r  r  y  sb   ø„ ð! #ð !°3ð !Àð !Ècð !Ð]aõ !ð°U·\±\ð ÀeÇlÁló ð  U§\¡\ð °e·l±l÷ r’   r  ÚnumberrÝ   c                 ó$  — g }| dz  dk(  r|j                  d«       | dz  } | dz  dk(  rŒt        dt        j                  | «      dz   d«      D ]*  }| |z  dk(  sŒ|j                  |«       | |z  } | |z  dk(  rŒŒ, | dkD  r|j                  | «       |S )Nr   r   r   rs   )Úappendrƒ   ÚmathÚisqrt)r  Úfactorsrõ   s      r�   Úprime_factorsr!  ž  s£   € Ø€Gà
�1‰*˜Š/Ø�‰�qÔØ�1‰ˆð �1‰*˜‹/ô �1”d—j‘j Ó(¨1Ñ,¨aÓ0ò ˆØ�q‰j˜A‹oØ�N‰N˜1ÔØ�q‰LˆFð �q‰j˜AŒoðð
 �‚zØ�‰�vÔØ€Nr’   r¶   rµ   r³   Ú
n_channelsc           	      ó°  — t        j                  t        j                  t        |«      ddd…   |¬«      d¬«      }t        j                  t        j                  t        | «      ddd…   |¬«      d¬«      }t        j                  |dz  |z  dz  «      j                  «       dz  }t        j                  |d   dz  |z  |z  «      j                  «       dz  }t        j                  ddd|gg|¬«      }	t        j                  t        j                  |«      |||gd¬«      }
t        j                  |t        j                  ||d   «      t        j                  ||d   «      |gd¬«      }t        j                  |	|
|gd¬«      }t        j                  |dd…dd…f   d¬«      j                  «       }||z  | z  |z  }t        j                  dt        j                  t        j                  ||¬«      «      |dz   |¬«      }t        j                  |j                  d«      t        j                  |«      j                  d«      z
  «      }||j                   d   k\  rt        j"                  |d¬«      }nDdd	lm}  ||j)                  «       j+                  «       «      \  }}t        j                  ||¬«      }d|d<   |j                   d   dz
  |d<   ||   S )
a  
    Plan out the dimensions for each layer in the HMLP encoder.

    This function determines the progression of dimensions (temporal, height, width, channels)
    for a multi-layer perceptual model that processes image/video patches. It follows these
    principles:
    1. Start with small dimensions and increase to full size
    2. Expand spatial dimensions (height/width) first, then temporal
    3. Increase channel count to avoid information bottlenecks
    4. Round channel dimensions to multiples of 64 for hardware efficiency

    The function computes optimal assignments of scale configurations to layers using either:
    - For n_layers >= len(scales): Individual best matching scales for each layer (allowing duplicates)
    - For n_layers < len(scales): Global optimal assignment via linear_sum_assignment

    The first and last scales are always fixed to ensure the proper input and output dimensions.

    Args:
        temporal_patch_size: Temporal dimension of input patches
        patch_size: Spatial dimension (height/width) of input patches
        n_layers: Number of layers in the encoder
        n_channels: Number of input channels (default: 3 for RGB)

    Returns:
        torch.LongTensor of shape `(n_layers + 1, 4)` where the last dim holds values for (t, h, w, c) grids.
    Nrß   r"  r   rX  r   rN   rs   )Úlinear_sum_assignment)r×   ÚcumprodÚtensorr!  Úceilrœ   ÚstackÚ	ones_likeÚ	full_liker^  Úprodrž   Úlinspacer*  ÚabsrŠ  rã   ÚargminÚscipy.optimizer$  ÚcpuÚnumpy)r¶   rµ   r³   r"  r#  ÚhÚtÚh_chÚt_chÚbaseÚspatialÚtemporalÚscalesÚsize_reductionÚtotal_elementsÚlog_ideal_scalesÚcost_matrixÚidxsr$  r�  Úidxs_nps                        r�   Úplan_out_scalesr@  ¯  s]  € ô: 	�‰”e—l‘l¤=°Ó#<¹T¸r¸TÑ#BÈ6ÔRÐXYÔZ€AÜ�‰”e—l‘l¤=Ð1DÓ#EÁdÈÀdÑ#KÐTZÔ[ÐabÔc€Aä�:‰:�a˜‘d˜ZÑ'¨"Ñ,Ó-×1Ñ1Ó3°bÑ8€DÜ�:‰:�a˜‘e˜q‘j :Ñ-°Ñ1Ó2×6Ñ6Ó8¸2Ñ=€Dä�<‰<˜!˜Q  :Ð.Ð/¸Ô?€DÜ�k‰kœ5Ÿ?™?¨1Ó-¨q°!°TÐ:ÀÔB€GÜ�{‰{˜AœuŸ™¨q°!°B±%Ó8¼%¿/¹/È!ÈQÈrÉUÓ:SÐUYÐZÐ`aÔb€HÜ�Y‰Y˜˜g xÐ0°aÔ8€Fä—Z‘Z ¢q¨#¨2¨# v¡°AÔ6×<Ñ<Ó>€Nà *Ñ,Ð/BÑBÀZÑO€NÜ—~‘~Ø	Œ5�9‰9”U—\‘\ .¸Ô@ÓAÀ8ÈaÁ<ÐX^ôÐô —)‘)Ð,×6Ñ6°qÓ9¼E¿I¹IÀnÓ<U×<_Ñ<_Ð`aÓ<bÑbÓc€Kà�6—<‘< ‘?Ò"Ü�|‰|˜K¨QÔ/‰å8á*¨;¯?©?Ó+<×+BÑ+BÓ+DÓE‰
ˆˆ7Ü�|‰|˜G¨FÔ3ˆð €Dˆ�GØ�|‰|˜A‰ Ñ"€Dˆ�HØ�$‰<Ðr’   c                   óf   ‡ — e Zd Zdefˆ fd„Zdej                  dee   dej                  fd„Z	ˆ xZ
S )ÚInklingVisionModelr  c                 óÊ  •— t         ‰	| �  |«       t        |j                  |j                  |j
                  |j                  «      | _        t        j                  «       | _
        t        t        | j                  d d | j                  dd  «      «      D ]£  \  }\  }}|d   |d   z  |d   |d   z  z  |d   |d   z  z  }||j
                  dz
  k(  r|j                  n|d   }|d   |d   z  }|d   |d   z  }| j                  j                  t        |d   |z  |||||j
                  dz
  k7  ¬«      «       Œ¥ t!        |j                  «      | _        | j%                  «        y )Nrß   rs   r   r   r   )r  r  r  r  r  )r‡   rÔ   r@  r¶   rµ   rM   r·   r9  rÕ   rÐ  Úencoder_layersrà  Úzipr«   r  r  rÏ   Ú
final_normrÓ  )
r‰   r  r‹   Ústart_scaleÚ	end_scaleÚshuffle_multr  r  r  rŒ   s
            €r�   rÔ   zInklingVisionModel.__init__î  s{  ø€ Ü‰Ñ˜Ô Ü%Ø×&Ñ&Ø×ÑØ×$Ñ$Ø×Ñó	
ˆŒô !Ÿm™m›oˆÔÜ+4´S¸¿¹ÀSÀbÐ9IÈ4Ï;É;ÐWXÐWYÈ?Ó5[Ó+\ò 	Ñ'ˆAÑ'�˜Yà˜1‘ ¨Q¡Ñ/°I¸a±LÀKÐPQÁNÑ4RÑSÐW`ÐabÑWcÐgrÐstÑguÑWuÑvð ð 56¸×9QÑ9QÐTUÑ9UÒ4U˜×0Ò0Ð[dÐefÑ[gˆJØ ‘l k°!¡nÑ4ˆGØ˜q‘\ [°¡^Ñ3ˆFØ×Ñ×&Ñ&Ü)Ø)¨!™n¨|Ñ;Ø)Ø#Ø!Ø &×":Ñ":¸QÑ">Ñ>ôõð	ô" )¨×)@Ñ)@ÓAˆŒØ�‰Õr’   Úpixel_valuesrŠ   rÝ   c                 ó¸   — |j                   d   }|}| j                  D ]  } ||¬«      }Œ | j                  |«      }|j                  |d«      }t	        ||¬«      S )Nr   )r8   rß   rÿ  )rã   rD  rF  r-  r   )r‰   rJ  rŠ   Únum_patchesr8   Úlayers         r�   rê   zInklingVisionModel.forward  sk   € Ø"×(Ñ(¨Ñ+ˆØ$ˆØ×(Ñ(ò 	?ˆEÙ!°Ô>‰Mð	?ð Ÿ™¨Ó6ˆØ%×-Ñ-¨k¸2Ó>ˆÜ)Ø+Ø'ô
ð 	
r’   )r“   r”   r•   r°   rÔ   r×   rë   r   r   rê   r£   r¤   s   @r�   rB  rB  í  s;   ø„ ðÐ2õ ð>
 E§L¡Lð 
¸FÐCUÑ<Vð 
Ð[`×[gÑ[g÷ 
r’   rB  zz
    The Base Inkling model which consists of a vision backbone and a language model without language modeling head.,
    ©Úcustom_introc                   óª  ‡ — e Zd ZdZdefˆ fd„Ze ed¬«      dej                  de
e   deez  fd	„«       «       Ze ed
¬«      	 ddej                  dej                   dz  deez  fd„«       «       Zdej                  dej                  dej                  defd„Zee	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                   dz  dej                   dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  de
e   deez  fd„«       «       Zˆ xZS )ÚInklingModelFr  c                 ó  •— t         ‰| �  |«       |j                  j                  | _        t	        |j                  «      | _        t        |j                  «      | _        t        |j                  «      | _        | j                  «        y rÓ   )r‡   rÔ   r,   rH   rÌ  Úlanguage_modelrú  r¨   Úaudio_towerrB  r²   Úvision_towerrÓ  rC  s     €r�   rÔ   zInklingModel.__init__$  sf   ø€ Ü‰Ñ˜Ô Ø ×,Ñ,×7Ñ7ˆŒÜ.¨v×/AÑ/AÓBˆÔÜ,¨V×-@Ñ-@ÓAˆÔÜ.¨v×/CÑ/CÓDˆÔØ�‰Õr’   zOProjects the last hidden state from the vision model into language model space.rN  rJ  rŠ   rÝ   c                 ó*   —  | j                   dd|i|¤ŽS )NrJ  r�   )rU  ©r‰   rJ  rŠ   s      r�   Úget_image_featureszInklingModel.get_image_features,  s   € ð
 !ˆt× Ñ ÑE¨lÐE¸fÑEÐEr’   zCProjects discretized dMel bin tokens into the language model space.Nrý  Úaudio_input_ids_maskc                 óŽ   — |�||j                  «          }n|j                  d|j                  d   «      }| j                  |«      S )aì  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
            Batch of (padded) dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) frames. When provided, only valid frames are encoded so that the
            number of returned audio embeddings matches the number of audio placeholder tokens.
        rß   )r¡   r-  rã   rT  )r‰   rý  rY  s      r�   Úget_audio_featureszInklingModel.get_audio_features3  sM   € ð  Ð+Ø-Ð.B×.GÑ.GÓ.IÑJ‰Oà-×5Ñ5°b¸/×:OÑ:OÐPRÑ:SÓTˆOØ×Ñ Ó0Ð0r’   r6   r7   ÚfeaturesÚtoken_idc                 óÚ  — |€Y| | j                  «       t        j                  |t        j                  |j                  ¬«      «      k(  }|j                  d«      }n||k(  }|j                  «       }|j                  d«      j                  |«      j                  |j                  «      }t        ||   j                  «       |j                  «       k(  d|› d|j                  d   › �«       |S )zö
        Obtains a multimodal placeholder mask from `input_ids` or `inputs_embeds` for the given `token_id`, and checks
        that the placeholder token count matches the length of `features`. If the lengths differ, an error is raised.
        )rô   r#  rß   zAMultimodal features and placeholder tokens do not match, tokens: z, features: r   )Úget_input_embeddingsr×   r&  Úlongr#  Úallrs  rŠ  Ú	expand_asrü   r   Únumelrã   )r‰   r6   r7   r\  r]  Úspecial_maskÚn_tokenss          r�   Úget_placeholder_maskz!InklingModel.get_placeholder_maskG  sä   € ð ÐØ(Ð,G¨D×,EÑ,EÓ,GÜ—‘˜X¬U¯Z©ZÀ×@TÑ@TÔUó-ñ ˆLð (×+Ñ+¨BÓ/‰Là$¨Ñ0ˆLà×#Ñ#Ó%ˆØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆÜØ˜,Ñ'×-Ñ-Ó/°8·>±>Ó3CÑCØOÐPXÈzÐYeÐfn×ftÑftÐuvÑfwÐexÐyô	
ð Ðr’   r9   rÔ  r  Útoken_type_idsrì  rÕ  Ú	lm_kwargsc           	      óÞ  — |du |	duz  rt        d«      ‚|	€/| j                  j                   | j                  «       |«      «      }	|�{| j	                  |«      j
                  }|j                  |	j                  |	j                  «      }| j                  ||	|| j                  j                  «      }|	j                  ||«      }	d}|�|| j                  ||«      j                  }|j                  |	j                  |	j                  «      }| j                  ||	|| j                  j                  «      }|	j                  ||«      }	t!        |x}t"        «      sA| j                  j%                  «       |	|||dœ}t'        di |¤Žt)        di |¤Žt+        di |¤Ždœ} | j                  d||||	|dœ|¤Ž}t-        |j                  |j.                  |j0                  |j2                  |�¬«      S d¬«      S )a/  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("google/inkling2-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/inkling2-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr×  rØ  rÙ  )r9   rÔ  r  r7   rÕ  )rÝ  r  r8   r¯  Úimage_hidden_statesr�   )rÞ  rS  rÑ  r_  rX  r   rü   r#  rô   rf  r  r¼   Úmasked_scatterr[  rÝ  r½   rÃ   rÄ   r´  r   r   r   rÉ   r  r8   r¯  )r‰   r6   rJ  rý  rY  r9   rÔ  r  rg  r7   rì  rÕ  rh  Úimage_featuresÚspecial_image_maskÚaudio_featuresÚspecial_audio_maskrâ  rã  rô  s                       r�   rê   zInklingModel.forwardb  s!  € ðd ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×/Ñ/×:Ñ:Ð;V¸4×;TÑ;TÓ;VÐW`Ó;aÓbˆMð Ð#Ø!×4Ñ4°\ÓB×PÑPˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø˜=¨.¸$¿+¹+×:TÑ:Tó"Ðð *×8Ñ8Ð9KÈ^Ó\ˆMð ˆØÐ&Ø!×4Ñ4°_ÐFZÓ[×mÑmˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø˜=¨.¸$¿+¹+×:TÑ:Tó"Ðð *×8Ñ8Ð9KÈ^Ó\ˆMô °Ð?Ð-ÄÔFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%UÜ$CÑ$RÀkÑ$Rñ#Ðð &�$×%Ñ%ð 
Ø.Ø%Ø+Ø'Øñ
ð ñ
ˆô *Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J ô
ð 	
ð
 QUô
ð 	
r’   rÓ   )NNNNNNNNNNN)r“   r”   r•   Úaccepts_loss_kwargsrº   rÔ   r   r   r×   rç  r   r   r;  r   rX  ræ  rë   r[  rœ   rf  r   r¡   rÉ   rê   r£   r¤   s   @r�   rQ  rQ    s@  ø„ ð  Ðð˜}õ ð ÙÐ!rÔsðFØ!×-Ñ-ðFØ9?Ð@RÑ9SðFà	Ð+Ñ	+òFó tó ðFð
 ÙÐ!fÔgð 59ñ1à×)Ñ)ð1ð $Ÿl™l¨TÑ1ð1ð 
Ð+Ñ	+ò	1ó hó ð1ð$à×#Ñ#ðð ×(Ñ(ðð ×#Ñ#ð	ð
 óð6 Øð .2Ø15Ø37Ø48Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ñh
à×#Ñ# dÑ*ðh
ð ×'Ñ'¨$Ñ.ðh
ð ×)Ñ)¨DÑ0ð	h
ð
 $Ÿl™l¨TÑ1ðh
ð Ÿ™ tÑ+ðh
ð ×&Ñ&¨Ñ-ðh
ð  ™ðh
ð ×(Ñ(¨4Ñ/ðh
ð ×(Ñ(¨4Ñ/ðh
ð × Ñ  4Ñ'ðh
ð ˜$‘;ðh
ð Ð.Ñ/ðh
ð 
Ð+Ñ	+òh
ó ó ôh
r’   rQ  c                   óô  ‡ — e Zd Zi ZddiZdZdefˆ fd„Zede	j                  dee   fd„«       Zee	 	 	 	 	 	 	 	 	 	 	 dd
e	j                  d	z  de	j                  d	z  de	j                   d	z  de	j                  d	z  ded	z  de	j                  d	z  de	j                   d	z  de	j                  d	z  de	j                  d	z  ded	z  dee	j                   z  dee   deez  fd„«       «       Z	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚInklingForConditionalGenerationrê  rë  Fr  c                 óî   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  j                  |j                  j                  d¬«      | _	        | j                  «        y r	  )r‡   rÔ   rQ  r®  rÕ   r  r,   rK   rH   rê  rÓ  rC  s     €r�   rÔ   z(InklingForConditionalGeneration.__init__Ü  sU   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
ô —y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Õr’   rJ  rŠ   c                 ó<   —  | j                   j                  |fi |¤ŽS rÓ   )r®  rX  rW  s      r�   rX  z2InklingForConditionalGeneration.get_image_featuresä  s   € à,ˆt�z‰z×,Ñ,¨\ÑD¸VÑDÐDr’   Nr6   r9   rÔ  r  rý  rY  r7   rì  rÕ  rí  rÝ   c                 ó6  —  | j                   d|||||||||
|	dœ
|¤Ž}|d   | j                  j                  j                  z  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  j                  }|�||j                  d   k  r	|dd|…f   }d}|	�# | j                  d||	|j                  d   dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )	a€	  
        audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
            Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
        audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
            Mask marking valid (non-padding) audio frames in `audio_input_ids`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, InklingForConditionalGeneration

        >>> model = InklingForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        )
r6   rJ  rý  rY  r9   rÔ  r  r7   rÕ  rì  r   Nrß   .rï  )rñ  rð  r  r8   r¯  rj  r�   )r®  r  r,   r>   rÃ   rœ   rò  rê  rI   rã   ró  rÍ   r  r8   r¯  rj  )r‰   r6   rJ  r9   rÔ  r  rý  rY  r7   rì  rÕ  rí  rŠ   rô  r8   rõ  rð  rI   rñ  s                      r�   rê   z'InklingForConditionalGeneration.forwardè  sF  € ðD �$—*‘*ð 
ØØ%Ø+Ø!5Ø)Ø%Ø+Ø'ØØñ
ð ñ
ˆð   ™
 T§[¡[×%<Ñ%<×%XÑ%XÑXˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ"Ÿk™k×5Ñ5×IÑIÐØÐ*Ð/BÀVÇ\Á\ÐRTÑEUÒ/UØ˜CÐ!5Ð"5Ð!5Ð5Ñ6ˆFàˆØÐØ%�4×%Ñ%Ðj¨V¸FÈvÏ|É|Ð\^ÑO_ÑjÐciÑjˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r’   c                 ó^   •— t        ‰| �  |f|||||	|
|dœ|¤Ž}|s|	s||d<   ||d<   ||d<   |S )N)r  r7   r9   rÔ  rÕ  rí  Úis_first_iterationrJ  rý  rY  )r‡   Úprepare_inputs_for_generation)r‰   r6   r  r7   rÔ  rJ  r9   rý  rY  rÕ  rí  rì  rw  rŠ   Úmodel_inputsrŒ   s                  €r�   rx  z=InklingForConditionalGeneration.prepare_inputs_for_generationM  sh   ø€ ô" ‘wÑ<Øð

à+Ø'Ø)Ø%ØØ)Ø1ñ

ð ñ

ˆñ ¡YØ+7ˆL˜Ñ(Ø.=ˆLÐ*Ñ+Ø3GˆLÐ/Ñ0àÐr’   )NNNNNNNNNNr   )NNNNNNNTNNF)r“   r”   r•   rö  r÷  rp  rº   rÔ   r   r×   rç  r   r   rX  r   ræ  rë   r   r¡   rœ   r;  rÍ   rê   rx  r£   r¤   s   @r�   rr  rr  Ï  sÉ  ø„ ð ÐØÐ0Ð1€Hð  Ðð˜}õ ð ðE¨u×/@Ñ/@ð EÈFÐSeÑLfò Eó ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø37Ø48Ø26Ø*.Ø!%Ø-.ña
à×#Ñ# dÑ*ða
ð ×'Ñ'¨$Ñ.ða
ð Ÿ™ tÑ+ð	a
ð
 ×&Ñ&¨Ñ-ða
ð  ™ða
ð ×)Ñ)¨DÑ0ða
ð $Ÿl™l¨TÑ1ða
ð ×(Ñ(¨4Ñ/ða
ð × Ñ  4Ñ'ða
ð ˜$‘;ða
ð ˜eŸl™lÑ*ða
ð Ð+Ñ,ða
ð 
Ð.Ñ	.òa
ó ó ða
ðL ØØØØØØ!ØØØØ ÷"ñ "r’   rr  )rº   r*   r¦   r°   r­  rÌ  ré  rú  rB  rQ  rr  )rn   Nr:  )r0  )lr  Úcollections.abcr   r×   Útorch.nnrÕ   Útorch.nn.functionalrù   rZ  Úhuggingface_hub.dataclassesr   Ú r   rµ  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Ú
generationr   Úintegrationsr   r   Úintegrations.accelerater   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úgemma3.modeling_gemma3r    r!   r"   r#   Ú&higgs_audio_v2.modeling_higgs_audio_v2r$   Úllama.modeling_llamar%   r&   Úmixtral.modeling_mixtralr'   Úqwen3_next.modeling_qwen3_nextr(   Ú
get_loggerr“   Úloggerr*   r¦   r°   rº   rÉ   rÍ   rÏ   ÚModulerÑ   rë   rž   r  r  r=  rJ  rO  ro  r{  rÖ   r    r‚  rŽ  r  r¢  r­  rÌ  ré  rº  rú  r  rœ   rŸ   r!  ræ  r@  rB  rQ  rr  Ú__all__r�   r’   r�   ú<module>r–     s7  ðó  Ý $ã Ý ß Ð Ý .å &Ý !ß .Ý 3Ý )ß IÝ =ß sÑ sÝ 9ß Sß FÝ &÷õ õ 8Ý 5÷ó õ Lß :Ý 5Ý Ið 
ˆ×	Ñ	˜HÓ	%€ð ôyÐ(ó yó ðyðx ô$Ð)ó $ó ð$ð  ô$Ð*ó $ó ð$ð  ô+(Ð$ó +(ó ð+(ô\	Ð!:ô 	ô	Ð$@ô 	ô	�\ô 	ô^˜BŸI™Iô ^ðB Ø)-ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð —<‘< $Ñ&ó%ô8a)�r—y‘yô a)ôH1�ô 1ô=�^ô =ô#H˜Ÿ	™	ô #HôL%˜2Ÿ9™9ô %ô8�—‘ô ñ( Ð0Ó1ð
 !%Ø!ñ'Ø—<‘<ð'à—‘ð'ð �L‰Lð'ð �,‰,˜Ñ
ð	'ð
 �d‘
ò'ó 2ð'ñ& Ð,Ó-ð !%Ø!ñ	'Ø—<‘<ð'à�L‰Lð'ð �,‰,˜Ñ
ð'ð �d‘
ò	'ó .ð'ñ, Ð*Ð,<Ð=Ó>ô;˜bŸi™ió ;ó ?ð;ô|.Ð4ô .ðb ô.˜_ó .ó ð.ðb ôL
Ð-ó L
ó ðL
ô^/
Ð*ô /
ôd ?Ð"8Ô >ô
Ð.ô 
ô" §	¡	ô "ðJ˜#ð  $ s¡)ó ð$ W\ñ;Øð;Ø*-ð;Ø9<ð;ØJMð;à
×Ñó;ô|+
Ð/ô +
ñ\ ðôô
l
Ð)ó l
óð
l
ñ^ ðôô
[Ð&<¸oó [óð
[ò|�r’   