
    ^jG                        d dl mZmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddl m!Z!  G d dejD                        Z# G d dejD                        Z$	 	 d.dejD                  dejJ                  dejJ                  dejJ                  dejJ                  dz  de&dz  de&dee   fdZ' G d dejD                        Z( G d d ejD                        Z) G d! d"e      Z*e G d# d$e             Z+ G d% d&ejD                        Z,e G d' d(e+             Z- ed)*       G d+ d,e+             Z.g d-Z/y)/    )CallableIterableN   )initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring	torch_int)can_return_tuplemerge_with_config_defaults)capture_outputs   )IJepaConfigc                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )IJepaPatchEmbeddingsz
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    configc                    t         |           |j                  }|j                  }t	        |t
              r|n||f}t	        |t
              r|n||f}|d   |d   z  |d   |d   z  z  | _        || _        || _        |j                  | _        t        j                  |j                  |j                  ||      | _        y )Nr   r   )kernel_sizestride)super__init__
image_size
patch_size
isinstancer   num_patchesnum_channelsnnConv2dhidden_size
projection)selfr   r   r    	__class__s       s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ijepa/modeling_ijepa.pyr   zIJepaPatchEmbeddings.__init__    s    &&
&&
#-j(#CZ*V`Ia
#-j(#CZ*V`Ia
&qMZ]:z!}PZ[\P]?]^$$"//))F$7$79K9KYclvw    pixel_valuesreturnc                     |j                   d   }|| j                  k7  rt        d| j                   d| d      | j                  |      j	                  d      j                  dd      S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .   )shaper#   
ValueErrorr'   flatten	transpose)r(   r,   r#   s      r*   forwardzIJepaPatchEmbeddings.forward-   su    #))!,4,,,!../yaI  |,44Q7AA!QGGr+   )
__name__
__module____qualname____doc__r   r   torchTensorr5   __classcell__r)   s   @r*   r   r      s4    x{ xHELL HU\\ Hr+   r   c            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  dej                  dz  dedej                  fdZ xZS )IJepaEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r   use_mask_tokenr-   Nc                    t         |           |r4t        j                  t	        j
                  dd|j                              nd | _        t        |      | _	        | j                  j                  }t        j                  t	        j                  d||j                              | _        t        j                  |j                        | _        |j                   | _        | j                  j"                  | _        y )Nr   )r   r   r$   	Parameterr:   zerosr&   
mask_tokenr   patch_embeddingsr"   randnposition_embeddingsDropouthidden_dropout_probdropoutr    r   )r(   r   r@   r"   r)   s       r*   r   zIJepaEmbeddings.__init__<   s    Q_",,u{{1a9K9K'LMei 4V <++77#%<<A{FL^L^0_#` zz&"<"<= ++//::r+   
embeddingsheightwidthc                 0   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  }|j                   d   }|| j
                  z  }|| j
                  z  }	t        |dz        }
|j                  d|
|
|      }|j                  dddd      }t        j                  j                  |||	fdd	      }|j                  dddd      j                  dd|      }|S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   g      ?r   r   r0   bicubicF)sizemodealign_corners)r1   rG   r:   jit
is_tracingr    r   reshapepermuter$   
functionalinterpolateview)r(   rK   rL   rM   r"   num_positionspatch_pos_embeddim
new_height	new_widthsqrt_num_positionss              r*   interpolate_pos_encodingz(IJepaEmbeddings.interpolate_pos_encodingF   s#    !&&q)0066q9 yy##%+*F6UZ?+++22r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nr+   r,   bool_masked_posra   c                     |j                   \  }}}}| j                  |      }|Z|j                   d   }	| j                  j                  ||	d      }
|j	                  d      j                  |
      }|d|z
  z  |
|z  z   }|r|| j                  |||      z   }ne|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d	      || j                  z   }| j                  |      }|S )	Nr   rO   g      ?r   zInput image size (*z) doesn't match model (z).)r1   rE   rD   expand	unsqueezetype_asra   r   r2   rG   rJ   )r(   r,   rb   ra   
batch_size_rL   rM   rK   
seq_lengthmask_tokensmasks               r*   r5   zIJepaEmbeddings.forwardm   s6    (4'9'9$
Avu**<8
&#))!,J//00ZLK",,R088ED#sTz2[45GGJ $#d&C&CJPVX]&^^J++u8J/J (% 9+,Adooa.@-AE  $d&>&>>J\\*-
r+   )F)NF)r6   r7   r8   r9   r   boolr   r:   r;   intra   
BoolTensorr5   r<   r=   s   @r*   r?   r?   7   s    ;{ ;D ;T ;%5<< % %UX %]b]i]i %T 48).	ll ))D0 #'	
 
r+   r?   modulequerykeyvalueattention_maskscalingrJ   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |dt        j                        j                  |j                        }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )NrO         r0   r   )r]   dtype)ptrainingr   )rQ   r:   matmulr4   r$   rX   softmaxfloat32tory   rJ   r{   
contiguous)
rp   rq   rr   rs   rt   ru   rJ   rv   attn_weightsattn_outputs
             r*   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r+   c                        e Zd Zdef fdZ	 d	dej                  dej                  dz  dee   de	ej                  ej                  f   fdZ
 xZS )
IJepaAttentionr   c                 (   t         |           || _        |j                  | _        t	        |d|j
                  |j                  z        | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  d      | _        y )Nhead_dimrx   F)biasT)r   r   r   num_attention_headsgetattrr&   r   attention_probs_dropout_probattention_dropoutru   	is_causalr$   Linearqkv_biasq_projk_projv_projo_projr(   r   r)   s     r*   r   zIJepaAttention.__init__   s,   #)#=#= 
F4F4F&JdJd4de!'!D!D}}d*ii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii : :T]] JFL^L^eijr+   Nhidden_statesrt   rv   r-   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  | j                  d|\  }
} |
j                  g |d j!                         }
| j#                  |
      }
|
|fS )NrO   r   r0           )rJ   ru   )r1   r   r   rZ   r4   r   r   r   get_interfacer   _attn_implementationr   r{   r   ru   rV   r   r   )r(   r   rt   rv   input_shapehidden_shapequery_states
key_statesvalue_statesattention_interfacer   r   s               r*   r5   zIJepaAttention.forward   sK    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r+   N)r6   r7   r8   r   r   r:   r;   r   r   tupler5   r<   r=   s   @r*   r   r      sf    k{ k" /3)||) t+) +,	)
 
u||U\\)	*)r+   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )IJepaMLPr   c                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y r   )r   r   r   r   
hidden_actactivation_fnr$   r   r&   intermediate_sizefc1fc2r   s     r*   r   zIJepaMLP.__init__   sd    #F$5$5699V//1I1IJ99V55v7I7IJr+   r   r-   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   )r(   r   s     r*   r5   zIJepaMLP.forward   s4    /**=9/r+   	r6   r7   r8   r   r   r:   r;   r5   r<   r=   s   @r*   r   r      s,    K{ KU\\ ell r+   r   c            	            e Zd Zdef fdZ	 d	dej                  dej                  dz  dee   dej                  fdZ	 xZ
S )

IJepaLayerr   c                 j   t         |           t        |      | _        t	        j
                  |j                  |j                        | _        t	        j
                  |j                  |j                        | _	        t        |      | _        t	        j                  |j                        | _        y )Neps)r   r   r   	attentionr$   	LayerNormr&   layer_norm_epslayernorm_beforelayernorm_afterr   mlprH   rI   rJ   r   s     r*   r   zIJepaLayer.__init__   sz    '/ "V-?-?VEZEZ [!||F,>,>FDYDYZF#zz&"<"<=r+   Nr   rt   rv   r-   c                     |}| j                  |      } | j                  ||fi |\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S r   )r   r   rJ   r   r   )r(   r   rt   rv   residualri   s         r*   r5   zIJepaLayer.forward   s     !--m<)4>>-R6Rq]3%0 !,,];/]3%0r+   r   )r6   r7   r8   r   r   r:   r;   r   r   r5   r<   r=   s   @r*   r   r      sV    >{ > /3|| t+ +,	
 
r+   r   c                        e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZdZeedZd	Z ej(                          fd
       Z xZS )IJepaPreTrainedModelr   ijepar,   )imageTr?   r   )r   
attentionsrE   c                 $   t         |   |       t        |t        j                  t        j
                  f      rct        j                  |j                  d| j                  j                         |j                   t        j                  |j                         yyt        |t              rct        j                  |j                  d| j                  j                         |j                   t        j                  |j                         yyy)zInitialize the weightsr   )meanstdN)r   _init_weightsr!   r$   r   r%   inittrunc_normal_weightr   initializer_ranger   zeros_r?   rG   rD   )r(   rp   r)   s     r*   r   z"IJepaPreTrainedModel._init_weights  s     	f%fryy"))45v}}3DKK<Y<YZ{{&FKK( '0v99IfIfg  ,F--. - 1r+   )r6   r7   r8   r   __annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_can_compile_fullgraphr   r   _can_record_outputs_input_embed_layerr:   no_gradr   r<   r=   s   @r*   r   r   
  sx    $O!&*#*L9N"&!#$ ,U]]_
/ 
/r+   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )IJepaPoolerr   c                     t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        y r   )
r   r   r$   r   r&   pooler_output_sizedenser   
pooler_act
activationr   s     r*   r   zIJepaPooler.__init__,  s>    YYv1163L3LM
 !2!23r+   r   r-   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r   )r(   r   first_token_tensorpooled_outputs       r*   r5   zIJepaPooler.forward1  s6     +1a40

#566r+   r   r=   s   @r*   r   r   +  s*    4{ 4
U\\ ell r+   r   c                        e Zd Zddededef fdZe ed      e	 	 	 	 dde	j                  dz  d	e	j                  dz  d
edz  de	j                  dz  dee   defd                     Z xZS )
IJepaModelFr   add_pooling_layerr@   c                    t         |   |       || _        t        ||      | _        t        j                  t        |j                        D cg c]  }t        |       c}      | _
        t        j                  |j                  |j                        | _        |rt        |      nd| _        | j#                          yc c}w )z
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )r@   r   N)r   r   r   r?   rK   r$   
ModuleListrangenum_hidden_layersr   layersr   r&   r   	layernormr   pooler	post_init)r(   r   r   r@   ri   r)   s        r*   r   zIJepaModel.__init__<  s     	 )&PmmvG_G_A`$aAZ%7$abf&8&8f>S>ST->k&)D	 %bs   C)tie_last_hidden_statesNr,   rb   ra   rt   rv   r-   c                    | j                   j                  j                  j                  j                  }|j                  |k7  r|j                  |      }| j                  |||      }t        | j                  ||      }|}| j                  D ]  }	 |	||fi |} | j                  |      }
| j                  | j                  |
      nd}t        |
|      S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )rb   ra   )r   inputs_embedsrt   N)last_hidden_statepooler_output)rK   rE   r'   r   ry   r   r   r   r   r   r   r
   )r(   r,   rb   ra   rt   rv   expected_dtypeembedding_outputr   layersequence_outputr   s               r*   r5   zIJepaModel.forwardL  s      99DDKKQQ/'??>:L??/Tl + 
 3;;*)

 )[[ 	KE!-J6JM	K ..78<8OO4UY)O[hiir+   )FF)NNNN)r6   r7   r8   r   rm   r   r   r   r   r:   r;   ro   r   r   r
   r5   r<   r=   s   @r*   r   r   :  s    { t ]a    E2 -13704.2 jllT) j ))D0 j #'+	 j
 t+ j +, j 
$ j  3   jr+   r   a  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )custom_introc                        e Zd Zdef fdZee	 	 	 d
dej                  dz  dej                  dz  de	dz  de
e   def
d	              Z xZS )IJepaForImageClassificationr   c                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r   r   )r   r   
num_labelsr   r   r$   r   r&   Identity
classifierr   r   s     r*   r   z$IJepaForImageClassification.__init__  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r+   Nr,   labelsra   rv   r-   c                     | j                   |fd|i|}|j                  }| j                  |j                  d            }d}| | j                  ||| j
                  fi |}t        |||j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ra   r   )r]   N)losslogitsr   r   )	r   r   r   r   loss_functionr   r   r   r   )	r(   r,   r   ra   rv   outputsr   r   r   s	            r*   r5   z#IJepaForImageClassification.forward  s      /9djj/
%=/
 /

 "33!5!5!!5!<=%4%%ffdkkLVLD$!//))	
 	
r+   )NNN)r6   r7   r8   r   r   r   r   r:   r;   rm   r   r   r   r5   r<   r=   s   @r*   r   r   r  s    
{ 
  -1&*04	
llT)
 t#
 #'+	

 +,
 

  
r+   r   )r   r   r   )Nr   )0collections.abcr   r   r:   torch.nnr$    r   r   activationsr   masking_utilsr   modeling_layersr	   modeling_outputsr
   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   utils.output_capturingr   configuration_ijepar   Moduler   r?   r;   floatr   r   r   r   r   r   r   r   __all__ r+   r*   <module>r     s   /   & ! 6 9 Q F & B B I 5 ,H299 H<Sbii Sx !%II%<<% 
% <<	%
 LL4'% T\% % '(%8.)RYY .)bryy  + @ /? / /@"))  4j% 4j 4jn .
"6 .
.
b Pr+   