
    ^jM                     |   d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddl m!Z! e G d de             Z" G d dejF                        Z$ G d dejF                        Z% G d dejF                        Z& G d dejF                        Z'	 	 d6dejF                  dejP                  dejP                  dejP                  dejP                  dz  d e)dz  d!e)d"ee   fd#Z* G d$ d%ejF                        Z+ G d& d'ejF                        Z, G d( d)ejF                        Z- G d* d+ejF                        Z. G d, d-ejF                        Z/ G d. d/e      Z0e G d0 d1e             Z1 G d2 d3e1      Z2e G d4 d5e1             Z3d5d1gZ4y)7    )Callable)	dataclassN)nn   )initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)merge_with_config_defaults)capture_outputs   )RadioConfigc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   y)RadioModelOutputa  Output of [`RadioModel`].

    Args:
        summary (`torch.FloatTensor` of shape `(batch_size, num_summary_idxs * hidden_size)`):
            Flattened summary embedding, gathered from the cls tokens selected by `config.summary_idxs`.
        features (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`):
            Dense spatial patch features.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Full token sequence (prefix tokens + patches) from the final encoder layer.
        hidden_states (`tuple[torch.FloatTensor]`, *optional*, returned when `output_hidden_states=True`):
            Tuple of `(batch_size, sequence_length, hidden_size)` tensors, one for the embedding output plus one for
            each encoder layer.
        attentions (`tuple[torch.FloatTensor]`, *optional*, returned when `output_attentions=True`):
            Tuple of `(batch_size, num_heads, sequence_length, sequence_length)` attention weights, one per layer.
    Nsummaryfeatureslast_hidden_statehidden_states
attentions)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   r   r   tupler        s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/radio/modeling_radio.pyr   r   (   s}      )-GU%,)-He$&-26u((4/659M5**+d2926Je''(4/6r&   r   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )RadioInputConditionerzFNormalizes pixel values; arithmetic is done in float32 then cast back.configc                 *   t         |           | j                  dt        j                  |j
                        j                  ddd      d       | j                  dt        j                  |j                        j                  ddd      d       y )N	norm_meanr   T
persistentnorm_std)super__init__register_bufferr!   tensorr,   viewr0   selfr*   	__class__s     r'   r2   zRadioInputConditioner.__init__D   sx    [%,,v7G7G*H*M*MbRSUV*WdhiZfoo)F)K)KBPQST)Ubfgr&   pixel_valuesreturnc                     |j                         | j                  j                         z
  | j                  j                         z  }|j                  |j                        S N)floatr,   r0   todtype)r7   r9   
normalizeds      r'   forwardzRadioInputConditioner.forwardI   sI    "((*T^^-A-A-CCt}}GZGZG\\
}}\//00r&   
r   r   r   r    r   r2   r!   TensorrA   __classcell__r8   s   @r'   r)   r)   A   s/    Ph{ h
1ELL 1U\\ 1r&   r)   c                        e Zd ZdZdef fdZdej                  dej                  fdZde	e
e
f   dej                  dej                  fd	Zdej                  dej                  fd
Z xZS )RadioPatchEmbeddingszCropped Position Embedding (CPE) patch generator.

    Splits the image into patches, projects them, adds a resolution-interpolated
    absolute position embedding, and prepends learned cls + register tokens.
    r*   c                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _        |j                  |j                  z  | _	        | j                  | j                  z  }t        j                  |j                  |j                  dz  z  |j                  d      | _        t        j                  t        j                   d||j                              | _        t        j                  t        j                   |j
                  |j                  z   |j                              | _        y )N   Fbiasr   )r1   r2   
patch_sizehidden_size	embed_dimnum_cls_tokensnum_registersmax_img_sizemax_rowsmax_colsr   Linearnum_channelspatch_projection	Parameterr!   zerosposition_embeddingcls_register_token)r7   r*   num_positionsr8   s      r'   r2   zRadioPatchEmbeddings.__init__U   s    ++++$33#11++v/@/@@++v/@/@@5 "		&*=*=@Q@QST@T*TV\VhVhot u"$,,u{{1mVM_M_/`"a"$,,KK--0D0DDfFXFXY#
r&   r9   r:   c                     | j                   }|j                  \  }}}}||z  ||z  }}|j                  ||||||      }	|	j                  dddddd      j                  |||z  ||z  |z        }	|	S )Nr   rI      r   r      )rL   shapereshapepermute)
r7   r9   psbatchchannelsheightwidthrowscolspatchess
             r'   _image_to_patchesz&RadioPatchEmbeddings._image_to_patchesf   s    __)5););&xr\5B;d&&uhb$K//!Q1a3;;E4$;PX[]P]`bPbcr&   
input_dimsr?   c                    | j                   j                  d| j                  | j                  d      j	                  dddd      }t        |      }t        j                  |j                         ||fdd      j                  |      }|d   |j                  d	   k  r|d
d |d   d d f   }|d   |j                  d   k  r|d
d d d |d   f   }|j                  d	d  t        |      k7  r?t        j                  |j                         t        |      dd      j                  |      }|j                  d      j	                  ddd      S )Nr   r-   r   r   rI   bilinearF)sizemodealign_corners.)rY   r`   rR   rS   ra   maxFinterpolater=   r>   r_   r$   flatten)r7   rk   r?   posmax_dims        r'   _interpolate_position_embeddingz4RadioPatchEmbeddings._interpolate_position_embeddingn   s*   %%--arRZZ[\^_abdefj/mmCIIKw.@zafgjjkpqa=399R=(c?Z]?A-.Ca=399R=(c1o
1o-.C99RS>U:..--		%
2C*dijmmnstC{{1~%%aA..r&   c                    | j                  | j                  |            }|j                  d   | j                  z  |j                  d   | j                  z  f}|| j	                  ||j
                        z   }| j                  j                  d      j                  |j                  d   dd      }t        j                  ||gd      S )Nrq   r-   r   r   dim)rV   rj   r_   rL   rx   r?   rZ   	unsqueezeexpandr!   cat)r7   r9   ri   rk   prefixs        r'   rA   zRadioPatchEmbeddings.forwardz   s    ''(>(>|(LM"((,?ASASTVAW[_[j[jAjk
D@@W]][[((2215<<W]]1=MrSUVyy&'*22r&   )r   r   r   r    r   r2   r!   rC   rj   r$   intr?   rx   rA   rD   rE   s   @r'   rG   rG   N   sz    
{ 
"ell u|| 
/%S/ 
/RWR]R] 
/bgbnbn 
/3ELL 3U\\ 3r&   rG   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )RadioMLPr:   c                 ~   t         |           |j                  x}}t        |j                  |j                  z        }t        j                  ||d      | _        t        |j                  t              rt        |j                     | _        n|j                  | _        t        j                  ||d      | _        y )NTrJ   )r1   r2   rM   r   	mlp_ratior   rT   fc1
isinstance
hidden_actstrr   
activationfc2r7   r*   in_featuresout_featureshidden_featuresr8   s        r'   r2   zRadioMLP.__init__   s    %+%7%77lf0063C3CCD99[/Ef''-$V%6%67DO$//DO99_lFr&   hidden_statec                 l    | j                  |      }| j                  |      }| j                  |      }|S r<   )r   r   r   r7   r   s     r'   rA   zRadioMLP.forward   s2    xx-|4xx-r&   r:   Nr   r   r   r2   r!   rC   rA   rD   rE   s   @r'   r   r      s$    	GELL U\\ r&   r   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )RadioLayerScaler:   c                     t         |           t        j                  |j                  t        j                  |j                        z        | _        y r<   )	r1   r2   r   rW   layerscale_valuer!   onesrM   lambda1r6   s     r'   r2   zRadioLayerScale.__init__   s8    ||F$;$;ejjI[I[>\$\]r&   r   c                      || j                   z  S r<   )r   r   s     r'   rA   zRadioLayerScale.forward   s    dll**r&   r   r   rE   s   @r'   r   r      s$    ^+ELL +U\\ +r&   r   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |d      }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr-         rI   r   rz   )ptrainingr   )
rn   r!   matmul	transposer   
functionalsoftmaxr   r   
contiguous)
r   r   r   r   r   r   r   r   attn_weightsattn_outputs
             r'   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$r&   c                        e Zd Zdef fdZdej                  dee   de	ej                  ej                  f   fdZ
 xZS )RadioSelfAttentionr*   c                 2   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        y )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .r   FrJ   )r1   r2   rM   num_attention_headshasattr
ValueErrorr*   r   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr   rT   qkv_biasr   r   r   r6   s     r'   r2   zRadioSelfAttention.__init__   sF    : ::a?PVXhHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r&   r   r   r:   c                    |j                   d   }|d| j                  | j                  f} | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      }t        j                  | j                  j                  t              } || |||d f| j                  | j                  | j                  sdn| j                   d|\  }	}
|	j#                         d d | j$                  fz   }|	j'                  |      }	|	|
fS )Nr   r-   r   rI           )r   r   r   rq   )r_   r   r   r   r5   r   r   r   r   get_interfacer*   _attn_implementationr   r   r   r   r   rn   r   r`   )r7   r   r   
batch_size	new_shape	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapes               r'   rA   zRadioSelfAttention.forward   sY   
 #((+
D$<$<d>V>VV	0DHH]+00)<FFq!L	4djj/44i@JJ1aP4djj/44i@JJ1aP(?(M(MKK,,.E)
 *=
*
 nnLL#}}C$2C2C
*
 
*
& #0"4"4"6s";t?Q?Q>S"S%--.EFo--r&   )r   r   r   r   r2   r!   rC   r   r   r$   rA   rD   rE   s   @r'   r   r      sN    ]{ ](.||. +,. 
u||U\\)	*	.r&   r   c                   x     e Zd ZdZdef fdZdej                  dej                  dej                  fdZ xZ	S )RadioSelfOutputz
    The residual connection is defined in RadioLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r*   c                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y r<   )	r1   r2   r   rT   rM   denseDropouthidden_dropout_probr   r6   s     r'   r2   zRadioSelfOutput.__init__   sB    YYv1163E3EF
zz&"<"<=r&   r   input_tensorr:   c                 J    | j                  |      }| j                  |      }|S r<   )r   r   )r7   r   r   s      r'   rA   zRadioSelfOutput.forward   s$    

=1]3r&   rB   rE   s   @r'   r   r      s=    
>{ >
U\\  RWR^R^ r&   r   c                   f     e Zd Zdef fdZdej                  dee   dej                  fdZ	 xZ
S )RadioAttentionr*   c                 b    t         |           t        |      | _        t	        |      | _        y r<   )r1   r2   r   	attentionr   outputr6   s     r'   r2   zRadioAttention.__init__  s&    +F3%f-r&   r   r   r:   c                 V     | j                   |fi |\  }}| j                  ||      }|S r<   )r   r   )r7   r   r   self_attn_output_r   s         r'   rA   zRadioAttention.forward  s5    
 -dnn]EfE!-}=r&   )r   r   r   r   r2   r!   rC   r   r   rA   rD   rE   s   @r'   r   r     s>    .{ .
|| +, 
	r&   r   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )RadioSwiGLUFFNr:   c                 0   t         |           |j                  x}}t        |j                  |j                  z        }t        |dz  dz        dz   dz  dz  }t        j                  |d|z  d      | _        t        j                  ||d      | _        y )NrI   r         TrJ   )	r1   r2   rM   r   r   r   rT   
weights_inweights_outr   s        r'   r2   zRadioSwiGLUFFN.__init__  s    %+%7%77lf0063C3CCD2Q67!;AAE))K_1D4P99_lNr&   r   c                     | j                  |      }|j                  dd      \  }}t        j                  j	                  |      |z  }| j                  |      S )NrI   r-   rz   )r   chunkr   r   silur   )r7   r   x1x2hiddens        r'   rA   zRadioSwiGLUFFN.forward  sS    |4##A2#.B##B'",''r&   r   r   rE   s   @r'   r   r     s$    O(ELL (U\\ (r&   r   c                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
RadioDropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probr:   Nc                 0    t         |           || _        y r<   )r1   r2   r   )r7   r   r8   s     r'   r2   zRadioDropPath.__init__-  s    "r&   r   c                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )Nr   r   r   )r   )r?   device)
r   r   r_   ndimr!   randr?   r   floordiv)r7   r   	keep_probr_   random_tensors        r'   rA   zRadioDropPath.forward1  s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r&   c                      d| j                    S )Nzp=)r   r7   s    r'   
extra_reprzRadioDropPath.extra_repr:  s    DNN#$$r&   )r   )r   r   r   r    r=   r2   r!   rC   rA   r   r   rD   rE   s   @r'   r   r   &  sB    #% #$ #<U\\ <ell <%C %r&   r   c                   d     e Zd ZdZdeddf fdZdej                  dej                  fdZ xZ	S )
RadioLayerzCThis corresponds to the Block class in the original implementation.r*   r:   Nc                    t         |           t        j                  |j                  |j
                        | _        t        |      | _        t        |      | _
        |j                  dkD  rt        |j                        nt        j                         | _        t        j                  |j                  |j
                        | _        |j                   rt#        |      | _        nt'        |      | _        t        |      | _        y )N)epsr   )r1   r2   r   	LayerNormrM   layer_norm_epsnorm1r   r   r   layer_scale1drop_path_rater   Identity	drop_pathnorm2use_swiglu_ffnr   mlpr   layer_scale2r6   s     r'   r2   zRadioLayer.__init__A  s    \\&"4"4&:O:OP
'/+F3AGAVAVY\A\v'<'<=bdbmbmbo\\&"4"4&:O:OP
  %f-DH'DH+F3r&   r   c                 "   | j                  |      }| j                  |      }| j                  |      }| j                  |      |z   }| j	                  |      }| j                  |      }| j                  |      }| j                  |      |z   }|S r<   )r   r   r   r   r  r  r  )r7   r   hidden_states_normself_attention_outputlayer_outputs        r'   rA   zRadioLayer.forwardQ  s     "ZZ6 $/A B $ 1 12G H '<=M zz-0xx-((6 ~~l3mCr&   rB   rE   s   @r'   r   r   >  s8    M4{ 4t 4 || 
r&   r   c                   h    e Zd ZeZdZdZdZdgZdgZ	dZ
dZeedZ ej                          d        Zy)	RadioPreTrainedModelmodelr9   Tr   zlayer_scale\d+\.lambda1)r   r   c                 :   | j                   j                  }t        |t        j                        rOt        j                  |j                  d|       |j                   t        j                  |j                         y y t        |t        j                        r?t        j                  |j                         t        j                  |j                         y t        |t              rEt        j                  |j                  d|       t        j                  |j                  d|       y t        |t              r5t        j                   |j"                  | j                   j$                         y t        |t&              rt        j(                  |j*                  t-        j.                  | j                   j*                        j1                  ddd             t        j(                  |j2                  t-        j.                  | j                   j2                        j1                  ddd             y t        |t4              rXt        j(                  |j6                  t-        j.                  | j                   j6                  t,        j8                               y y )Nr   )meanstdr-   r   r?   )r*   initializer_ranger   r   rT   inittrunc_normal_weightrK   zeros_r   ones_rG   rY   rZ   r   	constant_r   r   r)   copy_r,   r!   r4   r5   r0   
RadioModelsummary_idxslong)r7   r   r  s      r'   _init_weightsz"RadioPreTrainedModel._init_weightsv  s    kk++fbii(v}}3C@{{&FKK( '-KK$JJv}}% 45v88sLv88sL0NN6>>4;;+G+GH 56JJv''dkk6K6K)L)Q)QRTVWYZ)[\JJvT[[5I5I(J(O(OPRTUWX(YZ
+JJv**ELL9Q9QY^YcYc,de ,r&   N)r   r   r   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_keys_to_ignore_on_load_missing_supports_sdpa_supports_flash_attnr   r   _can_record_outputsr!   no_gradr  r%   r&   r'   r
  r
  g  s`    L$O&*#%'A&B#N#(
 U]]_f fr&   r
  c                   t     e Zd Zdef fdZe ed      dej                  de	e
   defd              Z xZS )	RadioEncoderr*   c                     t         |   |       t        j                  t	        |j
                        D cg c]  }t        |       c}      | _        | j                          y c c}w r<   )	r1   r2   r   
ModuleListrangenum_hidden_layersr   layer	post_init)r7   r*   r   r8   s      r'   r2   zRadioEncoder.__init__  sK     ]]fF^F^@_#`1Jv$6#`a
 $as   A&F)tie_last_hidden_statesr   r   r:   c                 L    | j                   D ]
  } ||      } t        |      S )N)r   )r,  r
   )r7   r   r   r,  s       r'   rA   zRadioEncoder.forward  s,     ZZ 	1E!-0M	1??r&   )r   r   r   r   r2   r   r   r!   rC   r   r   r
   rA   rD   rE   s   @r'   r'  r'    sU    { 
  E2@U\\ @VDV=W @\k @ 3  @r&   r'  c                        e Zd Zdef fdZedefd       Zd Ze	e
dej                  dee   defd              Z xZS )	r  r*   c                 4   t         |   |       || _        t        |      | _        t        |      | _        t        |      | _        | j                  dt        j                  |j                  t        j                        d       | j                          y )Nr  r  Tr.   )r1   r2   r*   r)   input_conditionerrG   
embeddingsr'  encoderr3   r!   r4   r  r  r-  r6   s     r'   r2   zRadioModel.__init__  st     !6v!>.v6#F+^U\\&:M:MUZU_U_-`mqrr&   r:   c                 .    | j                   j                  S r<   )r*   rL   r   s    r'   rL   zRadioModel.patch_size  s    {{%%%r&   c                 P    | j                   }t        j                         | _         |S )zCDetach the input conditioner (caller applies normalization itself).)r2  r   r   )r7   conditioners     r'   make_preprocessor_externalz%RadioModel.make_preprocessor_external  s!    ,,!#r&   r9   r   c                    | j                  |      }| j                  |      } | j                  |fi |}|j                  }| j                  j
                  }|d d d | j                  j                  f   }|d d | j                  f   j                  d      }|d d |d f   }	t        ||	||j                  |j                        S )Nr   )r   r   r   r   r   )r2  r3  r4  r   r*   num_summary_tokensrO   r  ru   r   r   r   )
r7   r9   r   r   encoder_outputsr   num_skipall_summaryr   r   s
             r'   rA   zRadioModel.forward  s     --l;5+74<<+P+P+==;;11'+GT[[-G-G+G(GHa!2!223;;A>$Q	\2/)77&11
 	
r&   )r   r   r   r   r2   propertyr   rL   r8  r   r   r!   rC   r   r   r   rA   rD   rE   s   @r'   r  r    sk    {  &C & & 
ELL 
FCU<V 
[k 
  
r&   r  )Nr   )5collections.abcr   dataclassesr   r!   torch.nn.functionalr   r   rs    r   r  activationsr   modeling_layersr	   modeling_outputsr
   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   utils.output_capturingr   configuration_radior   r   Moduler)   rG   r   r   rC   r=   r   r   r   r   r   r   r   r
  r'  r  __all__r%   r&   r'   <module>rN     s  * % !     & ! 9 < F & I I 7 5 , 7{ 7 70
1BII 
113299 13hryy &+bii + !%II%<<% 
% <<	%
 LL4'% T\% % '(%:4. 4.pbii $RYY  (RYY ("%BII %0&+ &R #f? #f #fL@' @ '
% '
 '
T /
0r&   