Ë
    ÜÓ^j>+  ã                   ó  — d dl mZ ddlmZ ddlmZ ddlmZmZ  ej                  e
«      Z ed¬«      e G d„ d	e«      «       «       Z ed¬«      e G d
„ de«      «       «       Z ed¬«      e G d„ de«      «       «       Zg d¢Zy)é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringÚloggingz meta-llama/Llama-4-Scout-17B-16E)Ú
checkpointc                   óˆ  — e Zd ZU dZddddddddœZdZdZdZee	d	<   d
Z
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZeee   z  eeef   z  e	d<   dZeee   z  eeef   z  e	d<   dZee	d<   dZee	d<   dZee	d<   d Zee	d!<   d"Zee	d#<   d"Zee	d$<   d%Zee	d&<   d'Zeez  e	d(<   d'Zeez  e	d)<   d*Z e!e"z  d*z  e	d+<   y*),ÚLlama4VisionConfigaw  
    vision_output_dim (`int`, *optional*, defaults to 7680):
        Dimensionality of the vision model output. Includes output of transformer
        encoder with intermediate layers and global transformer encoder.
    pixel_shuffle_ratio (`float`, *optional*, defaults to 0.5):
        Pixel-shuffle ratio for downsampling patch tokens. Smaller values produce fewer tokens (more downsampling).
    projector_input_dim (`int`, *optional*, defaults to 4096):
        Width of the vision adapter MLP before pixel shuffle. Larger value increases capacity and compute.
    projector_output_dim (`int`, *optional*, defaults to 4096):
        Output width of the vision adapter. Larger value yields higher-dimensional image features.
    projector_dropout (`float`, *optional*, defaults to 0.0):
        Dropout rate inside the vision adapter MLP. Higher value adds more regularization.
    ÚcolwiseÚrowwiseÚcolwise_gather_output)zmodel.layers.*.self_attn.q_projzmodel.layers.*.self_attn.k_projzmodel.layers.*.self_attn.v_projzmodel.layers.*.self_attn.o_projzvision_adapter.mlp.fc1zvision_adapter.mlp.fc2zpatch_embedding.linearÚllama4_vision_modelÚvision_configi   Úhidden_sizeÚgeluÚ
hidden_acté"   Únum_hidden_layersé   Únum_attention_headsr   Únum_channelsi   Úintermediate_sizei   Úvision_output_dimiÀ  Ú
image_sizeé   Ú
patch_sizeçñhãˆµøä>Únorm_epsÚdefaultÚvision_feature_select_strategyç{®Gáz”?Úinitializer_rangeg      à?Úpixel_shuffle_ratioi   Úprojector_input_dimÚprojector_output_dimFÚmulti_modal_projector_biasç        Úprojector_dropoutÚattention_dropoutNÚrope_parameters)#Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úbase_model_tp_planÚ
model_typeÚbase_config_keyr   ÚintÚ__annotations__r   Ústrr   r   r   r   r   r   ÚlistÚtupler   r   Úfloatr!   r#   r$   r%   r&   r'   Úboolr)   r*   r+   r   Údict© ó    úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/llama4/configuration_llama4.pyr   r      s8  … ñð ,5Ø+4Ø+4Ø+4Ø"+Ø"+Ø"9ñÐð '€JØ%€Oà€K�ÓØ€J�ÓØÐ�sÓØ!Ð˜Ó!Ø€L�#ÓØ!Ð�sÓ!Ø!Ð�sÓ!Ø47€J��d˜3‘i‘ %¨¨S¨¡/Ñ1Ó7Ø46€J��d˜3‘i‘ %¨¨S¨¡/Ñ1Ó6Ø€HˆeÓØ*3Ð" CÓ3Ø#Ð�uÓ#Ø!$Ð˜Ó$Ø#Ð˜Ó#Ø $Ð˜#Ó$Ø',Ð Ó,Ø%(Ð�u˜s‘{Ó(Ø%(Ð�u˜s‘{Ó(Ø48€O�^ dÑ*¨TÑ1Ô8r<   r   c                   ó®  ‡ — e Zd ZU dZdZdgZdZdddddddddddddœZddddd	d	dddd
dœ
ZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZeed<   dZe
ed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Ze
d&z  ed'<   d(Ze
d&z  ed)<   d*Ze
ee
   z  d&z  ed+<   d,Zeed-<   d.Z ee
z  ed/<   d(Z!e
ed0<   d1Z"e
ed2<   d&Z#ee
   d&z  ed3<   d(Z$e
ed4<   d$Z%eed5<   d,Z&eed6<   d7Z'eed8<   d.Z(eed9<   d&Z)e*e+z  d&z  ed:<   d&Z,ee
   d&z  ed;<   d<Z-e
ed=<   dZ.e
d&z  ed><   d&Z/ee   d&z  ed?<   d$Z0eed@<   dZ1e
edA<   dBZ2eedC<   d,Z3eedD<   ˆ fdE„Z4ˆ xZ5S )FÚLlama4TextConfigaO  
    intermediate_size_mlp (`int`, *optional*, defaults to 16384):
        Intermediate size of dense MLP layers. Larger value increases FFN capacity and compute.
    moe_layers (`list[int]`, *optional*):
        List of layer indices that use MoE. Overrides `interleave_moe_layer_step` when set.
    interleave_moe_layer_step (`int`, *optional*, defaults to 1):
        Spacing between MoE layers when `moe_layers` is `None`. Larger value means fewer MoE layers.
    use_qk_norm (`bool`, *optional*, defaults to `True`):
        Whether to L2-normalize queries/keys on RoPE layers. Can stabilize attention when enabled.
    no_rope_layers (`list[int]`, *optional*):
        List with at least the same length as the number of layers in the model.
        A `1` at an index position indicates that the corresponding layer will use RoPE,
        while a `0` indicates that it's a NoPE layer.
    no_rope_layer_interval (`int`, *optional*, defaults to 4):
        If `no_rope_layers` is `None`, it will be created using a NoPE layer every
        `no_rope_layer_interval` layers.
    attention_chunk_size (`int`, *optional*, defaults to 8192):
        Chunk size for the attention computation. Smaller value enforces more local attention and lowers memory.
    attn_temperature_tuning (`bool`, *optional*, defaults to `True`):
        Whether to dynamically scale the attention temperature for each query token based on sequence length.
        Recommended for long sequences (e.g., >32k tokens) to maintain stable output results.
    floor_scale (`int`, *optional*, defaults to 8192):
        Base scale (in tokens) for attention temperature tuning. Larger value delays scaling to longer positions.
    attn_scale (`float`, *optional*, defaults to 0.1):
        Strength of attention temperature tuning. Larger value increases scaling at long positions.

    Example:
    Úllama4_textÚpast_key_valuesg    €„Ar   r   Úpacked_rowwise)úlayers.*.self_attn.q_projúlayers.*.self_attn.k_projúlayers.*.self_attn.v_projúlayers.*.self_attn.o_projz-layers.*.feed_forward.shared_expert.gate_projz+layers.*.feed_forward.shared_expert.up_projz-layers.*.feed_forward.shared_expert.down_projú*layers.*.feed_forward.experts.gate_up_projú'layers.*.feed_forward.experts.down_projúlayers.*.feed_forward.gate_projúlayers.*.feed_forward.up_projúlayers.*.feed_forward.down_projÚgrouped_gemmÚ	ep_router)
rC   rD   rE   rF   rG   rH   rI   rJ   rK   zlayers.*.feed_forward.routeri@ Ú
vocab_sizei   r   i    r   i @  Úintermediate_size_mlpé0   r   é(   r   é   Únum_key_value_headsé€   Úhead_dimÚsilur   i   Úmax_position_embeddingsr"   r#   r   Úrms_norm_epsTÚ	use_cacheNÚpad_token_idé   Úbos_token_idé   Úeos_token_idFÚtie_word_embeddingsr(   r*   Únum_experts_per_tokr   Únum_local_expertsÚ
moe_layersÚinterleave_moe_layer_stepÚuse_qk_normÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiser+   Úno_rope_layersé   Úno_rope_layer_intervalÚattention_chunk_sizeÚlayer_typesÚattn_temperature_tuningÚfloor_scalegš™™™™™¹?Ú
attn_scaleÚattention_biasc                 óž  •— | j                   €| j                  | _         t        | j                  «      D �cg c]   }t	        |dz   | j
                  z  dk7  «      ‘Œ" }}| j                  r| j                  n|| _        | j                  �| j                  n| j                  | j                  z  | _        | j                  �| j                  n6t        t        | j                  dz
  | j                  | j                  «      «      | _	        | j                  €#| j                  D �cg c]  }|rdnd‘Œ
 c}| _        t        ‰| �8  di |¤Ž y c c}w c c}w )Nr[   r   Úchunked_attentionÚfull_attentionr;   )rS   r   Úranger   r3   rj   rh   rU   r   rb   r6   rc   rl   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	layer_idxÚdefault_no_rope_layersÚno_ropeÚ	__class__s        €r=   rv   zLlama4TextConfig.__post_init__¯   sA  ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ô V[Ð[_×[qÑ[qÓUrö"
ØHQŒC�˜Q‘ $×"=Ñ"=Ñ=ÀÑBÕCð"
Ðð "
ð 6:×5HÒ5H˜d×1Ò1ÐNdˆÔØ)-¯©Ð)B˜ŸšÈ×HXÑHXÐ\`×\tÑ\tÑHtˆŒð �‰Ð*ð �OŠOäÜØ×2Ñ2°QÑ6Ø×*Ñ*Ø×2Ñ2óóð 	Œð ×ÑÐ#àTX×TgÑTgö ØIP¡wÑ#Ð4DÑDò ˆDÔô 	‰ÑÑ' Ó'ùò/"
ùò& s   ¶%EÄ E
)6r,   r-   r.   r/   r1   Úkeys_to_ignore_at_inferenceÚdefault_thetar0   Úbase_model_ep_planrN   r3   r4   r   r   rO   r   r   rS   rU   r   r5   rW   r#   r8   rX   rY   r9   rZ   r\   r^   r6   r_   r*   r`   ra   rb   rc   rd   re   rf   rg   r+   r   r:   rh   rj   rk   rl   rm   rn   ro   rp   rv   Ú__classcell__©r|   s   @r=   r?   r?   M   s/  ø… ñð: €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø9BØ7@Ø9BØ6FØ3<Ø+4Ø)2Ø+4ñÐð &/Ø%.Ø%.Ø%.Ø6DØ3AØ+4Ø)2Ø+4Ø(3ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!Ø!&Ð˜3Ó&ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€HˆcÓØ€J�ÓØ#,Ð˜SÓ,Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ#€L�#˜‘*Ó#Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø %Ð˜Ó%Ø%(Ð�u˜s‘{Ó(Ø Ð˜Ó ØÐ�sÓØ#'€J��S‘	˜DÑ Ó'Ø%&Ð˜sÓ&Ø€K�ÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø!$Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø'+€N�D˜‘I Ñ$Ó+Ø"#Ð˜CÓ#Ø'+Ð˜# ™*Ó+Ø$(€K��c‘˜TÑ!Ó(Ø$(Ð˜TÓ(Ø€K�ÓØ€J�ÓØ €N�DÓ ÷(ð (r<   r?   c                   ó²   ‡ — e Zd ZU dZdZddddœZeedœZdd	iZ	d
Z
eez  d
z  ed<   d
Zeez  d
z  ed<   dZeed<   dZeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚLlama4Configat  
    boi_token_index (`int`, *optional*, defaults to 200080):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 200081):
        The end-of-image token index to wrap the image prompt.

    ```python
    >>> from transformers import Llama4Model, Llama4Config

    >>> # Initializing a Llama4 7B style configuration
    >>> configuration = Llama4Config()

    >>> # Initializing a model from the Llama4 7B style configuration
    >>> model = Llama4Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úllama4Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configr   zmulti_modal_projector.linear_1Úcolwise_repNr   r‹   i� i‘ iœ Fr_   c                 ó¸  •— | j                   €%t        «       | _         t        j                  d«       n4t	        | j                   t
        «      rt        di | j                   ¤Ž| _         | j                  €%t        «       | _        t        j                  d«       n4t	        | j                  t
        «      rt        di | j                  ¤Ž| _        t        ‰| �$  di |¤Ž y )Nz9vision_config is None, using default llama4 vision configz5text_config is None, using default llama4 text configr;   )
r   r   ÚloggerÚinfoÚ
isinstancer:   r‹   r?   ru   rv   )rw   rx   r|   s     €r=   rv   zLlama4Config.__post_init__ö   s§   ø€ Ø×ÑÐ%Ü!3Ó!5ˆDÔÜ�K‰KÐSÕTÜ˜×*Ñ*¬DÔ1Ü!3Ñ!I°d×6HÑ6HÑ!IˆDÔà×ÑÐ#Ü/Ó1ˆDÔÜ�K‰KÐOÕPÜ˜×(Ñ(¬$Ô/Ü/ÑC°$×2BÑ2BÑCˆDÔÜ‰ÑÑ' Ó'r<   )r,   r-   r.   r/   r1   Úattribute_mapr?   r   Úsub_configsr0   r   r:   r   r4   r‹   r†   r3   r‡   r…   r_   r9   rv   r€   r�   s   @r=   rƒ   rƒ   Í   s™   ø… ñð( €Jà-Ø)Ø)ñ€Mð
 #3ÐEWÑX€Kà(¨-ðÐð 59€M�4Ð*Ñ*¨TÑ1Ó8Ø26€K�Ð(Ñ(¨4Ñ/Ó6Ø!€O�SÓ!Ø!€O�SÓ!Ø#Ð�sÓ#Ø %Ð˜Ó%÷(ð (r<   rƒ   )rƒ   r?   r   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú
get_loggerr,   rŽ   r   r?   rƒ   Ú__all__r;   r<   r=   ú<module>r™      s³   ðõ" /å 3Ý 1ß ,ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð=Ô>Øô-9Ð)ó -9ó ó ?ð-9ñ` Ð=Ô>Øô{(Ð'ó {(ó ó ?ð{(ñ| Ð=Ô>Øô3(Ð#ó 3(ó ó ?ð3(òl E�r<   