Ë
    ÜÓ^j¾M  ã                   ó|  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddl m!Z! e G d„ de«      «       Z" G d„ dejF                  «      Z$ G d„ dejF                  «      Z% G d„ dejF                  «      Z& G d„ dejF                  «      Z'	 	 d6dejF                  dejP                  dejP                  dejP                  dejP                  dz  d e)dz  d!e)d"ee   fd#„Z* G d$„ d%ejF                  «      Z+ G d&„ d'ejF                  «      Z, G d(„ d)ejF                  «      Z- G d*„ d+ejF                  «      Z. G d,„ d-ejF                  «      Z/ G d.„ d/e«      Z0e G d0„ d1e«      «       Z1 G d2„ d3e1«      Z2e G d4„ d5e1«      «       Z3d5d1gZ4y)7é    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚRadioConfigc                   óæ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   y)ÚRadioModelOutputa  Output of [`RadioModel`].

    Args:
        summary (`torch.FloatTensor` of shape `(batch_size, num_summary_idxs * hidden_size)`):
            Flattened summary embedding, gathered from the cls tokens selected by `config.summary_idxs`.
        features (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`):
            Dense spatial patch features.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Full token sequence (prefix tokens + patches) from the final encoder layer.
        hidden_states (`tuple[torch.FloatTensor]`, *optional*, returned when `output_hidden_states=True`):
            Tuple of `(batch_size, sequence_length, hidden_size)` tensors, one for the embedding output plus one for
            each encoder layer.
        attentions (`tuple[torch.FloatTensor]`, *optional*, returned when `output_attentions=True`):
            Tuple of `(batch_size, num_heads, sequence_length, sequence_length)` attention weights, one per layer.
    NÚsummaryÚfeaturesÚlast_hidden_stateÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   Útupler   © ó    ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/radio/modeling_radio.pyr   r   (   s}   … ñð  )-€GˆU×Ñ Ñ%Ó,Ø)-€Hˆe×Ñ $Ñ&Ó-Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r&   r   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚRadioInputConditionerzFNormalizes pixel values; arithmetic is done in float32 then cast back.Úconfigc                 ó*  •— t         ‰| �  «        | j                  dt        j                  |j
                  «      j                  ddd«      d¬«       | j                  dt        j                  |j                  «      j                  ddd«      d¬«       y )NÚ	norm_meanéÿÿÿÿr   T©Ú
persistentÚnorm_std)ÚsuperÚ__init__Úregister_bufferr!   Útensorr,   Úviewr0   ©Úselfr*   Ú	__class__s     €r'   r2   zRadioInputConditioner.__init__D   sx   ø€ Ü‰ÑÔØ×Ñ˜[¬%¯,©,°v×7GÑ7GÓ*H×*MÑ*MÈbÐRSÐUVÓ*WÐdhÐÔiØ×Ñ˜Z¬¯©°f·o±oÓ)F×)KÑ)KÈBÐPQÐSTÓ)UÐbfÐÕgr&   Úpixel_valuesÚreturnc                 óÄ   — |j                  «       | j                  j                  «       z
  | j                  j                  «       z  }|j                  |j                  «      S ©N)Úfloatr,   r0   ÚtoÚdtype)r7   r9   Ú
normalizeds      r'   ÚforwardzRadioInputConditioner.forwardI   sI   € Ø"×(Ñ(Ó*¨T¯^©^×-AÑ-AÓ-CÑCÀtÇ}Á}×GZÑGZÓG\Ñ\ˆ
Ø�}‰}˜\×/Ñ/Ó0Ð0r&   ©
r   r   r   r    r   r2   r!   ÚTensorrA   Ú__classcell__©r8   s   @r'   r)   r)   A   s/   ø„ ÙPðh˜{õ hð
1 E§L¡Lð 1°U·\±\÷ 1r&   r)   c                   óÞ   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zde	e
e
f   dej                  dej                  fd	„Zdej                  dej                  fd
„Zˆ xZS )ÚRadioPatchEmbeddingszÕCropped Position Embedding (CPE) patch generator.

    Splits the image into patches, projects them, adds a resolution-interpolated
    absolute position embedding, and prepends learned cls + register tokens.
    r*   c                 óæ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _        |j                  |j                  z  | _	        | j                  | j                  z  }t        j                  |j                  |j                  dz  z  |j                  d¬«      | _        t        j                  t        j                   d||j                  «      «      | _        t        j                  t        j                   |j
                  |j                  z   |j                  «      «      | _        y )Né   F©Úbiasr   )r1   r2   Ú
patch_sizeÚhidden_sizeÚ	embed_dimÚnum_cls_tokensÚnum_registersÚmax_img_sizeÚmax_rowsÚmax_colsr   ÚLinearÚnum_channelsÚpatch_projectionÚ	Parameterr!   ÚzerosÚposition_embeddingÚcls_register_token)r7   r*   Únum_positionsr8   s      €r'   r2   zRadioPatchEmbeddings.__init__U   s  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ×+Ñ+ˆŒØ$×3Ñ3ˆÔØ#×1Ñ1ˆÔà×+Ñ+¨v×/@Ñ/@Ñ@ˆŒØ×+Ñ+¨v×/@Ñ/@Ñ@ˆŒØŸ™¨¯©Ñ5ˆä "§	¡	¨&×*=Ñ*=À×@QÑ@QÐSTÑ@TÑ*TÐV\×VhÑVhÐotÔ uˆÔÜ"$§,¡,¬u¯{©{¸1¸mÈV×M_ÑM_Ó/`Ó"aˆÔÜ"$§,¡,Ü�K‰K˜×-Ñ-°×0DÑ0DÑDÀf×FXÑFXÓYó#
ˆÕr&   r9   r:   c                 óà   — | j                   }|j                  \  }}}}||z  ||z  }}|j                  ||||||«      }	|	j                  dddddd«      j                  |||z  ||z  |z  «      }	|	S )Nr   rI   é   r   r   é   )rL   ÚshapeÚreshapeÚpermute)
r7   r9   ÚpsÚbatchÚchannelsÚheightÚwidthÚrowsÚcolsÚpatchess
             r'   Ú_image_to_patchesz&RadioPatchEmbeddings._image_to_patchesf   s‡   € Ø�_‰_ˆØ)5×);Ñ);Ñ&ˆˆx˜ Ø˜r‘\ 5¨B¡;ˆdˆØ×&Ñ& u¨h¸¸bÀ$ÈÓKˆØ—/‘/ ! Q¨¨1¨a°Ó3×;Ñ;¸EÀ4È$Á;ÐPXÐ[]ÑP]Ð`bÑPbÓcˆØˆr&   Ú
input_dimsr?   c                 ó˜  — | j                   j                  d| j                  | j                  d«      j	                  dddd«      }t        |«      }t        j                  |j                  «       ||fdd¬«      j                  |«      }|d   |j                  d	   k  r|d
d |d   …d d …f   }|d   |j                  d   k  r|d
d d …d |d   …f   }|j                  d	d  t        |«      k7  r?t        j                  |j                  «       t        |«      dd¬«      j                  |«      }|j                  d«      j	                  ddd«      S )Nr   r-   r   r   rI   ÚbilinearF)ÚsizeÚmodeÚalign_cornerséþÿÿÿ.)rY   r`   rR   rS   ra   ÚmaxÚFÚinterpolater=   r>   r_   r$   Úflatten)r7   rk   r?   ÚposÚmax_dims        r'   Ú_interpolate_position_embeddingz4RadioPatchEmbeddings._interpolate_position_embeddingn   s*  € Ø×%Ñ%×-Ñ-¨a°·±ÀÇÁÈrÓR×ZÑZÐ[\Ð^_ÐabÐdeÓfˆÜ�j“/ˆÜ�m‰m˜CŸI™I›K¨w¸Ð.@ÀzÐafÔg×jÑjÐkpÓqˆØ�a‰=˜3Ÿ9™9 R™=Ò(Ø�c˜?˜Z¨™]˜?ªAÐ-Ñ.ˆCØ�a‰=˜3Ÿ9™9 R™=Ò(Ø�cš1˜o 
¨1¡˜oÐ-Ñ.ˆCØ�9‰9�R�Sˆ>œU :Ó.Ò.Ü—-‘- §	¡	£´%¸
Ó2CÈ*ÐdiÔj×mÑmÐnsÓtˆCØ�{‰{˜1‹~×%Ñ% a¨¨AÓ.Ð.r&   c                 ó”  — | j                  | j                  |«      «      }|j                  d   | j                  z  |j                  d   | j                  z  f}|| j	                  ||j
                  «      z   }| j                  j                  d«      j                  |j                  d   dd«      }t        j                  ||gd¬«      S )Nrq   r-   r   r   ©Údim)rV   rj   r_   rL   rx   r?   rZ   Ú	unsqueezeÚexpandr!   Úcat)r7   r9   ri   rk   Úprefixs        r'   rA   zRadioPatchEmbeddings.forwardz   s³   € Ø×'Ñ'¨×(>Ñ(>¸|Ó(LÓMˆØ"×(Ñ(¨Ñ,°·±Ñ?À×ASÑASÐTVÑAWÐ[_×[jÑ[jÑAjÐkˆ
Ø˜D×@Ñ@ÀÈWÏ]É]Ó[Ñ[ˆØ×(Ñ(×2Ñ2°1Ó5×<Ñ<¸W¿]¹]È1Ñ=MÈrÐSUÓVˆÜ�y‰y˜& 'Ð*°Ô2Ð2r&   )r   r   r   r    r   r2   r!   rC   rj   r$   Úintr?   rx   rA   rD   rE   s   @r'   rG   rG   N   sz   ø„ ñð
˜{õ 
ð"¨e¯l©lð ¸u¿|¹|ó ð
/¸%ÀÀSÀ¹/ð 
/ÐRW×R]ÑR]ð 
/Ðbg×bnÑbnó 
/ð3 E§L¡Lð 3°U·\±\÷ 3r&   rG   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚRadioMLPr:   c                 ó~  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        j                  ||d¬«      | _        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  ||d¬«      | _        y )NTrJ   )r1   r2   rM   r€   Ú	mlp_ratior   rT   Úfc1Ú
isinstanceÚ
hidden_actÚstrr   Ú
activationÚfc2©r7   r*   Úin_featuresÚout_featuresÚhidden_featuresr8   s        €r'   r2   zRadioMLP.__init__ƒ   s�   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ—9‘9˜[¨/ÀÔEˆŒÜ�f×'Ñ'¬Ô-Ü$ V×%6Ñ%6Ñ7ˆD�Oà$×/Ñ/ˆDŒOÜ—9‘9˜_¨lÀÔFˆ�r&   Úhidden_statec                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r<   )r…   r‰   rŠ   ©r7   r�   s     r'   rA   zRadioMLP.forwardŽ   s2   € Ø—x‘x Ó-ˆØ—‘ |Ó4ˆØ—x‘x Ó-ˆØÐr&   ©r:   N©r   r   r   r2   r!   rC   rA   rD   rE   s   @r'   r‚   r‚   ‚   s$   ø„ õ	Gð E§L¡Lð °U·\±\÷ r&   r‚   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚRadioLayerScaler:   c                 óª   •— t         ‰| �  «        t        j                  |j                  t        j                  |j                  «      z  «      | _        y r<   )	r1   r2   r   rW   Úlayerscale_valuer!   ÚonesrM   Úlambda1r6   s     €r'   r2   zRadioLayerScale.__init__–   s8   ø€ Ü‰ÑÔÜ—|‘| F×$;Ñ$;¼e¿j¹jÈ×I[ÑI[Ó>\Ñ$\Ó]ˆ�r&   r�   c                 ó    — || j                   z  S r<   )r™   r‘   s     r'   rA   zRadioLayerScale.forwardš   s   € Ø˜dŸl™lÑ*Ð*r&   r’   r“   rE   s   @r'   r•   r•   •   s$   ø„ õ^ð+ E§L¡Lð +°U·\±\÷ +r&   r•   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr-   ç      à¿rI   r   rz   )ÚpÚtrainingr   )
rn   r!   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr¡   r¦   Ú
contiguous)
r›   rœ   r�   rž   rŸ   r    r¡   r¢   Úattn_weightsÚattn_outputs
             r'   Úeager_attention_forwardr®   ž   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r&   c                   ó„   ‡ — e Zd Zdefˆ fd„Zdej                  dee   de	ej                  ej                  f   fd„Z
ˆ xZS )ÚRadioSelfAttentionr*   c                 ó2  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                   ¬«      | _        t        j                  |j                  | j                  |j                   ¬«      | _        t        j                  |j                  | j                  |j                   ¬«      | _        y )	Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads ú.r¤   FrJ   )r1   r2   rM   Únum_attention_headsÚhasattrÚ
ValueErrorr*   r€   Úattention_head_sizeÚall_head_sizeÚattention_probs_dropout_probÚdropout_probr    Ú	is_causalr   rT   Úqkv_biasrœ   r�   rž   r6   s     €r'   r2   zRadioSelfAttention.__init__¼   sF  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ" 6×#5Ñ#5Ð"6ð 7Ø×3Ñ3Ð4°Að7óð ð
 ˆŒØ#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ"×?Ñ?ˆÔØ×/Ñ/°Ñ5ˆŒØˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆ�
r&   r   r¢   r:   c                 ó°  — |j                   d   }|d| j                  | j                  f} | j                  |«      j                  |Ž j                  dd«      } | j                  |«      j                  |Ž j                  dd«      } | j                  |«      j                  |Ž j                  dd«      }t        j                  | j                  j                  t        «      } || |||d f| j                  | j                  | j                  sdn| j                   dœ|¤Ž\  }	}
|	j#                  «       d d | j$                  fz   }|	j'                  |«      }	|	|
fS )Nr   r-   r   rI   ç        )r»   r    r¡   rq   )r_   r´   r·   r�   r5   r¨   rž   rœ   r   Úget_interfacer*   Ú_attn_implementationr®   r»   r    r¦   rº   rn   r¸   r`   )r7   r   r¢   Ú
batch_sizeÚ	new_shapeÚ	key_layerÚvalue_layerÚquery_layerÚattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes               r'   rA   zRadioSelfAttention.forwardÐ   sY  € ð
 #×(Ñ(¨Ñ+ˆ
Ø  D×$<Ñ$<¸d×>VÑ>VÐVˆ	à0�D—H‘H˜]Ó+×0Ñ0°)Ð<×FÑFÀqÈ!ÓLˆ	Ø4�d—j‘j Ó/×4Ñ4°iÐ@×JÑJÈ1ÈaÓPˆØ4�d—j‘j Ó/×4Ñ4°iÐ@×JÑJÈ1ÈaÓPˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ *=ØØØØØð
*
ð —n‘nØ—L‘LØ#Ÿ}š}‘C°$×2CÑ2Cñ
*
ð ñ
*
Ñ&ˆ�ð #0×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×-Ñ-Ð.EÓFˆà˜oÐ-Ð-r&   )r   r   r   r   r2   r!   rC   r   r   r$   rA   rD   rE   s   @r'   r°   r°   »   sN   ø„ ð]˜{õ ]ð(.à—|‘|ð.ð Ð+Ñ,ð.ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	.r&   r°   c                   óx   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  fd„Zˆ xZ	S )ÚRadioSelfOutputz¢
    The residual connection is defined in RadioLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r*   c                 óÈ   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        y r<   )	r1   r2   r   rT   rM   ÚdenseÚDropoutÚhidden_dropout_probr¡   r6   s     €r'   r2   zRadioSelfOutput.__init__ù   sB   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r&   r   Úinput_tensorr:   c                 óJ   — | j                  |«      }| j                  |«      }|S r<   )rÍ   r¡   )r7   r   rÐ   s      r'   rA   zRadioSelfOutput.forwardþ   s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr&   rB   rE   s   @r'   rË   rË   ó   s=   ø„ ñð
>˜{õ >ð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r&   rË   c                   óf   ‡ — e Zd Zdefˆ fd„Zdej                  dee   dej                  fd„Z	ˆ xZ
S )ÚRadioAttentionr*   c                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y r<   )r1   r2   r°   Ú	attentionrË   Úoutputr6   s     €r'   r2   zRadioAttention.__init__  s&   ø€ Ü‰ÑÔÜ+¨FÓ3ˆŒÜ% fÓ-ˆ�r&   r   r¢   r:   c                 óV   —  | j                   |fi |¤Ž\  }}| j                  ||«      }|S r<   )rÕ   rÖ   )r7   r   r¢   Úself_attn_outputÚ_rÖ   s         r'   rA   zRadioAttention.forward  s5   € ð
 -˜dŸn™n¨]ÑE¸fÑEÑÐ˜!Ø—‘Ð-¨}Ó=ˆØˆr&   )r   r   r   r   r2   r!   rC   r   r   rA   rD   rE   s   @r'   rÓ   rÓ     s>   ø„ ð.˜{õ .ð
à—|‘|ðð Ð+Ñ,ðð 
�‰÷	r&   rÓ   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚRadioSwiGLUFFNr:   c                 ó0  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        |dz  dz  «      dz   dz  dz  }t        j                  |d|z  d¬«      | _        t        j                  ||d¬«      | _        y )NrI   r   é   é   TrJ   )	r1   r2   rM   r€   r„   r   rT   Ú
weights_inÚweights_outr‹   s        €r'   r2   zRadioSwiGLUFFN.__init__  sŠ   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ˜°Ñ2°QÑ6Ó7¸!Ñ;ÀÑAÀAÑEˆäŸ)™) K°°_Ñ1DÈ4ÔPˆŒÜŸ9™9 _°lÈÔNˆÕr&   r�   c                 ó¶   — | j                  |«      }|j                  dd¬«      \  }}t        j                  j	                  |«      |z  }| j                  |«      S )NrI   r-   rz   )rß   Úchunkr   r©   Úsilurà   )r7   r�   Úx1Úx2Úhiddens        r'   rA   zRadioSwiGLUFFN.forward  sS   € Ø—‘ |Ó4ˆØ×#Ñ# A¨2Ð#Ó.‰ˆˆBÜ—‘×#Ñ# BÓ'¨"Ñ,ˆØ×Ñ Ó'Ð'r&   r’   r“   rE   s   @r'   rÛ   rÛ     s$   ø„ õOð( E§L¡Lð (°U·\±\÷ (r&   rÛ   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚRadioDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr:   Nc                 ó0   •— t         ‰| �  «        || _        y r<   )r1   r2   ré   )r7   ré   r8   s     €r'   r2   zRadioDropPath.__init__-  s   ø€ Ü‰ÑÔØ"ˆ�r&   r   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )Nr¾   r   r   )r   )r?   Údevice)
ré   r¦   r_   Úndimr!   Úrandr?   rì   ÚfloorÚdiv)r7   r   Ú	keep_probr_   Úrandom_tensors        r'   rA   zRadioDropPath.forward1  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r&   c                 ó    — d| j                   › �S )Nzp=)ré   ©r7   s    r'   Ú
extra_reprzRadioDropPath.extra_repr:  s   € Ø�D—N‘NÐ#Ð$Ð$r&   )r¾   )r   r   r   r    r=   r2   r!   rC   rA   rˆ   rõ   rD   rE   s   @r'   rè   rè   &  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r&   rè   c                   ód   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )Ú
RadioLayerzCThis corresponds to the Block class in the original implementation.r*   r:   Nc                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        |«      | _
        |j                  dkD  rt        |j                  «      nt        j                  «       | _        t        j                  |j                  |j
                  ¬«      | _        |j                   rt#        |«      | _        nt'        |«      | _        t        |«      | _        y )N)Úepsr¾   )r1   r2   r   Ú	LayerNormrM   Úlayer_norm_epsÚnorm1rÓ   rÕ   r•   Úlayer_scale1Údrop_path_raterè   ÚIdentityÚ	drop_pathÚnorm2Úuse_swiglu_ffnrÛ   Úmlpr‚   Úlayer_scale2r6   s     €r'   r2   zRadioLayer.__init__A  s½   ø€ Ü‰ÑÔä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
Ü'¨Ó/ˆŒÜ+¨FÓ3ˆÔØAG×AVÑAVÐY\ÒA\œ v×'<Ñ'<Ô=Ôbd×bmÑbmÓboˆŒä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
à× Ò Ü% fÓ-ˆD�Hä Ó'ˆDŒHÜ+¨FÓ3ˆÕr&   r   c                 ó"  — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }|S r<   )rü   rÕ   rý   r   r  r  r  )r7   r   Úhidden_states_normÚself_attention_outputÚlayer_outputs        r'   rA   zRadioLayer.forwardQ  s”   € ð "ŸZ™Z¨Ó6ÐØ $§¡Ð/AÓ BÐØ $× 1Ñ 1Ð2GÓ HÐð Ÿ™Ð'<Ó=ÀÑMˆð —z‘z -Ó0ˆØ—x‘x Ó-ˆØ×(Ñ(¨Ó6ˆð —~‘~ lÓ3°mÑCˆàÐr&   rB   rE   s   @r'   r÷   r÷   >  s8   ø„ ÙMð4˜{ð 4¨tõ 4ð à—|‘|ðð 
�‰÷r&   r÷   c                   óh   — e Zd ZeZdZdZdZdgZdgZ	dZ
dZeedœZ ej                   «       d„ «       Zy)	ÚRadioPreTrainedModelÚmodelr9   Tr÷   zlayer_scale\d+\.lambda1)r   r   c                 ó:  — | j                   j                  }t        |t        j                  «      rOt        j                  |j                  d|¬«       |j                  � t        j                  |j                  «       y y t        |t        j                  «      r?t        j                  |j                  «       t        j                  |j                  «       y t        |t        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       y t        |t        «      r5t        j                   |j"                  | j                   j$                  «       y t        |t&        «      r±t        j(                  |j*                  t-        j.                  | j                   j*                  «      j1                  ddd«      «       t        j(                  |j2                  t-        j.                  | j                   j2                  «      j1                  ddd«      «       y t        |t4        «      rXt        j(                  |j6                  t-        j.                  | j                   j6                  t,        j8                  ¬«      «       y y )Nr¾   )ÚmeanÚstdr-   r   ©r?   )r*   Úinitializer_ranger†   r   rT   ÚinitÚtrunc_normal_ÚweightrK   Úzeros_rú   Úones_rG   rY   rZ   r•   Ú	constant_r™   r—   r)   Úcopy_r,   r!   r4   r5   r0   Ú
RadioModelÚsummary_idxsÚlong)r7   r›   r  s      r'   Ú_init_weightsz"RadioPreTrainedModel._init_weightsv  sŸ  € ð �k‰k×+Ñ+ˆÜ�fœbŸi™iÔ(Ü×Ñ˜vŸ}™}°3¸CÕ@Ø�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜¤§¡Ô-Ü�K‰K˜Ÿ™Ô$Ü�J‰J�v—}‘}Õ%Ü˜Ô 4Ô5Ü×Ñ˜v×8Ñ8¸sÈÕLÜ×Ñ˜v×8Ñ8¸sÈÖLÜ˜¤Ô0Ü�N‰N˜6Ÿ>™>¨4¯;©;×+GÑ+GÕHÜ˜Ô 5Ô6Ü�J‰J�v×'Ñ'¬¯©°d·k±k×6KÑ6KÓ)L×)QÑ)QÐRTÐVWÐYZÓ)[Ô\Ü�J‰J�v—‘¬¯©°T·[±[×5IÑ5IÓ(J×(OÑ(OÐPRÐTUÐWXÓ(YÕZÜ˜¤
Ô+Ü�J‰J�v×*Ñ*¬E¯L©L¸¿¹×9QÑ9QÔY^×YcÑYcÔ,dÕeð ,r&   N)r   r   r   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keys_to_ignore_on_load_missingÚ_supports_sdpaÚ_supports_flash_attnr÷   r°   Ú_can_record_outputsr!   Úno_gradr  r%   r&   r'   r
  r
  g  s`   „ à€LØÐØ$€OØ&*Ð#Ø%˜ÐØ'AÐ&BÐ#Ø€NØÐà#Ø(ñÐð
 €U‡]�]ƒ_ñfó ñfr&   r
  c                   ót   ‡ — e Zd Zdefˆ fd„Ze ed¬«      dej                  de	e
   defd„«       «       Zˆ xZS )	ÚRadioEncoderr*   c                 óÖ   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        | j                  «        y c c}w r<   )	r1   r2   r   Ú
ModuleListÚrangeÚnum_hidden_layersr÷   ÚlayerÚ	post_init)r7   r*   rÙ   r8   s      €r'   r2   zRadioEncoder.__init__�  sK   ø€ Ü‰Ñ˜Ô Ü—]‘]ÄÀf×F^ÑF^Ó@_Ö#`¸1¤J¨vÕ$6Ò#`ÓaˆŒ
Ø�‰Õùò $as   ·A&F)Útie_last_hidden_statesr   r¢   r:   c                 óL   — | j                   D ]
  } ||«      }Œ t        |¬«      S )N)r   )r,  r
   )r7   r   r¢   r,  s       r'   rA   zRadioEncoder.forward”  s,   € ð —Z‘Zò 	1ˆEÙ! -Ó0‰Mð	1ä°Ô?Ð?r&   )r   r   r   r   r2   r   r   r!   rC   r   r   r
   rA   rD   rE   s   @r'   r'  r'  Ž  sU   ø„ ð˜{õ ð
  Ù¨EÔ2ð@ U§\¡\ð @¸VÐDVÑ=Wð @Ð\kò @ó 3ó  ô@r&   r'  c                   ó‚   ‡ — e Zd Zdefˆ fd„Zedefd„«       Zd„ Ze	e
dej                  dee   defd„«       «       Zˆ xZS )	r  r*   c                 ó4  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        | j                  dt        j                  |j                  t        j                  ¬«      d¬«       | j                  «        y )Nr  r  Tr.   )r1   r2   r*   r)   Úinput_conditionerrG   Ú
embeddingsr'  Úencoderr3   r!   r4   r  r  r-  r6   s     €r'   r2   zRadioModel.__init__ž  st   ø€ Ü‰Ñ˜Ô ØˆŒÜ!6°vÓ!>ˆÔÜ.¨vÓ6ˆŒÜ# FÓ+ˆŒØ×Ñ˜^¬U¯\©\¸&×:MÑ:MÔUZ×U_ÑU_Ô-`ÐmqÐÔrØ�‰Õr&   r:   c                 ó.   — | j                   j                  S r<   )r*   rL   rô   s    r'   rL   zRadioModel.patch_size§  s   € à�{‰{×%Ñ%Ð%r&   c                 óP   — | j                   }t        j                  «       | _         |S )zCDetach the input conditioner (caller applies normalization itself).)r2  r   rÿ   )r7   Úconditioners     r'   Úmake_preprocessor_externalz%RadioModel.make_preprocessor_external«  s!   € à×,Ñ,ˆÜ!#§¡£ˆÔØÐr&   r9   r¢   c                 ó�  — | j                  |«      }| j                  |«      } | j                  |fi |¤Ž}|j                  }| j                  j
                  }|d d …d | j                  j                  …f   }|d d …| j                  f   j                  d«      }|d d …|d …f   }	t        ||	||j                  |j                  ¬«      S )Nr   )r   r   r   r   r   )r2  r3  r4  r   r*   Únum_summary_tokensrO   r  ru   r   r   r   )
r7   r9   r¢   r   Úencoder_outputsr   Únum_skipÚall_summaryr   r   s
             r'   rA   zRadioModel.forward±  sÉ   € ð ×-Ñ-¨lÓ;ˆØŸ™¨Ó5ˆØ+7¨4¯<©<¸Ñ+PÈÑ+PˆØ+×=Ñ=Ðà—;‘;×1Ñ1ˆØ'ªÐ+G¨T¯[©[×-GÑ-GÐ+GÐ(GÑHˆØša ×!2Ñ!2Ð2Ñ3×;Ñ;¸AÓ>ˆØ$¢Q¨©	 \Ñ2ˆäØØØ/Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r&   )r   r   r   r   r2   Úpropertyr€   rL   r8  r   r   r!   rC   r   r   r   rA   rD   rE   s   @r'   r  r  œ  sk   ø„ ð˜{õ ð ð&˜Cò &ó ð&òð Øð
 E§L¡Lð 
¸FÐCUÑ<Vð 
Ð[kò 
ó ó ô
r&   r  )Nr¾   )5Úcollections.abcr   Údataclassesr   r!   Útorch.nn.functionalr   r©   rs   Ú r   r  Úactivationsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_radior   r   ÚModuler)   rG   r‚   r•   rC   r=   r®   r°   rË   rÓ   rÛ   rè   r÷   r
  r'  r  Ú__all__r%   r&   r'   ú<module>rN     sÉ  ðõ* %Ý !ã ß Ð Ý å &Ý !Ý 9ß <ß FÝ &ß IÑ IÝ 7Ý 5Ý ,ð ô7�{ó 7ó ð7ô0
1˜BŸI™Iô 
1ô13˜2Ÿ9™9ô 13ôhˆr�y‰yô ô&+�b—i‘iô +ð !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô:4.˜Ÿ™ô 4.ôp�b—i‘iô ô$�R—Y‘Yô ô (�R—Y‘Yô (ô"%�B—I‘Iô %ô0&Ð+ô &ðR ô#f˜?ó #fó ð#fôL@Ð'ô @ð ô'
Ð%ó '
ó ð'
ðT Ð/Ð
0�r&   