Ë
    ÛÓ^j¤G  ã                   óÂ  — d dl mZmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddl m!Z!  G d„ dejD                  «      Z# G d„ dejD                  «      Z$	 	 d.dejD                  dejJ                  dejJ                  dejJ                  dejJ                  dz  de&dz  de&dee   fd„Z' G d„ dejD                  «      Z( G d„ d ejD                  «      Z) G d!„ d"e«      Z*e G d#„ d$e«      «       Z+ G d%„ d&ejD                  «      Z,e G d'„ d(e+«      «       Z- ed)¬*«       G d+„ d,e+«      «       Z.g d-¢Z/y)/é    )ÚCallableÚIterableNé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚIJepaConfigc                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚIJepaPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    Úconfigc                 ó„  •— t         ‰| �  «        |j                  }|j                  }t	        |t
        «      r|n||f}t	        |t
        «      r|n||f}|d   |d   z  |d   |d   z  z  | _        || _        || _        |j                  | _        t        j                  |j                  |j                  ||¬«      | _        y )Nr   r   )Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsÚnnÚConv2dÚhidden_sizeÚ
projection)Úselfr   r   r    Ú	__class__s       €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ijepa/modeling_ijepa.pyr   zIJepaPatchEmbeddings.__init__    sº   ø€ Ü‰ÑÔØ×&Ñ&ˆ
Ø×&Ñ&ˆ
Ü#-¨j¼(Ô#C‘ZÈ*ÐV`ÐIaˆ
Ü#-¨j¼(Ô#C‘ZÈ*ÐV`ÐIaˆ
à& q™M¨Z¸©]Ñ:¸zÈ!¹}ÐPZÐ[\ÑP]Ñ?]Ñ^ˆÔØ$ˆŒØ$ˆŒØ"×/Ñ/ˆÔÜŸ)™) F×$7Ñ$7¸×9KÑ9KÐYcÐlvÔwˆ�ó    Úpixel_valuesÚreturnc                 óÖ   — |j                   d   }|| j                  k7  rt        d| j                  › d|› d�«      ‚| j                  |«      j	                  d«      j                  dd«      S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaper#   Ú
ValueErrorr'   ÚflattenÚ	transpose)r(   r,   r#   s      r*   ÚforwardzIJepaPatchEmbeddings.forward-   su   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜðØ!×.Ñ.Ð/¨y¸¸ÀaðIóð ð �‰˜|Ó,×4Ñ4°QÓ7×AÑAÀ!ÀQÓGÐGr+   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚTensorr5   Ú__classcell__©r)   s   @r*   r   r      s4   ø„ ñðx˜{õ xðH E§L¡Lð H°U·\±\÷ Hr+   r   c            	       óÒ   ‡ — e Zd ZdZddededdfˆ fd„Zdej                  de	d	e	dej                  fd
„Z
	 	 ddej                  dej                  dz  dedej                  fd„Zˆ xZS )ÚIJepaEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r   Úuse_mask_tokenr-   Nc                 óú  •— t         ‰| �  «        |r4t        j                  t	        j
                  dd|j                  «      «      nd | _        t        |«      | _	        | j                  j                  }t        j                  t	        j                  d||j                  «      «      | _        t        j                  |j                  «      | _        |j                   | _        | j                  j"                  | _        y )Nr   )r   r   r$   Ú	Parameterr:   Úzerosr&   Ú
mask_tokenr   Úpatch_embeddingsr"   ÚrandnÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutr    r   )r(   r   r@   r"   r)   s       €r*   r   zIJepaEmbeddings.__init__<   s®   ø€ Ü‰ÑÔÙQ_œ"Ÿ,™,¤u§{¡{°1°a¸×9KÑ9KÓ'LÔMÐeiˆŒÜ 4°VÓ <ˆÔØ×+Ñ+×7Ñ7ˆÜ#%§<¡<´·±¸A¸{ÈF×L^ÑL^Ó0_Ó#`ˆÔ Ü—z‘z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØ×/Ñ/×:Ñ:ˆ�r+   Ú
embeddingsÚheightÚwidthc                 ó0  — |j                   d   }| j                  j                   d   }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  }|j                   d   }|| j
                  z  }|| j
                  z  }	t        |dz  «      }
|j                  d|
|
|«      }|j                  dddd«      }t        j                  j                  |||	fdd¬	«      }|j                  dddd«      j                  dd|«      }|S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   éÿÿÿÿg      à?r   r   r0   ÚbicubicF)ÚsizeÚmodeÚalign_corners)r1   rG   r:   ÚjitÚ
is_tracingr    r   ÚreshapeÚpermuter$   Ú
functionalÚinterpolateÚview)r(   rK   rL   rM   r"   Únum_positionsÚpatch_pos_embedÚdimÚ
new_heightÚ	new_widthÚsqrt_num_positionss              r*   Úinterpolate_pos_encodingz(IJepaEmbeddings.interpolate_pos_encodingF   s#  € ð !×&Ñ& qÑ)ˆØ×0Ñ0×6Ñ6°qÑ9ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×2Ñ2ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆàÐr+   r,   Úbool_masked_posra   c                 ó   — |j                   \  }}}}| j                  |«      }|�Z|j                   d   }	| j                  j                  ||	d«      }
|j	                  d«      j                  |
«      }|d|z
  z  |
|z  z   }|r|| j                  |||«      z   }ne|| j                  d   k7  s|| j                  d   k7  r2t        d|› d|› d| j                  d   › d| j                  d   › d�	«      ‚|| j                  z   }| j                  |«      }|S )	Nr   rO   g      ð?r   zInput image size (Ú*z) doesn't match model (z).)r1   rE   rD   ÚexpandÚ	unsqueezeÚtype_asra   r   r2   rG   rJ   )r(   r,   rb   ra   Ú
batch_sizeÚ_rL   rM   rK   Ú
seq_lengthÚmask_tokensÚmasks               r*   r5   zIJepaEmbeddings.forwardm   s6  € ð (4×'9Ñ'9Ñ$ˆ
�A�v˜uØ×*Ñ*¨<Ó8ˆ
àÐ&Ø#×)Ñ)¨!Ñ,ˆJØŸ/™/×0Ñ0°¸ZÈÓLˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJñ $Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^‰Jà˜Ÿ™¨Ñ+Ò+¨u¸¿¹ÈÑ8JÒ/JÜ Ø(¨¨°°%°ð 9ØŸ™¨Ñ+Ð,¨A¨d¯o©o¸aÑ.@Ð-AÀðEóð ð $ d×&>Ñ&>Ñ>ˆJà—\‘\ *Ó-ˆ
àÐr+   )F)NF)r6   r7   r8   r9   r   Úboolr   r:   r;   Úintra   Ú
BoolTensorr5   r<   r=   s   @r*   r?   r?   7   s–   ø„ ññ;˜{ð ;¸Dð ;ÈTõ ;ð%°5·<±<ð %Èð %ÐUXð %Ð]b×]iÑ]ió %ðT 48Ø).ñ	à—l‘lðð ×)Ñ)¨DÑ0ðð #'ð	ð
 
�‰÷r+   r?   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrJ   Úkwargsc                 óì  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )NrO   ç      à¿r0   r   )r]   Údtype)ÚpÚtrainingr   )rQ   r:   Úmatmulr4   r$   rX   ÚsoftmaxÚfloat32Útory   rJ   r{   Ú
contiguous)
rp   rq   rr   rs   rt   ru   rJ   rv   Úattn_weightsÚattn_outputs
             r*   Úeager_attention_forwardrƒ   �   sÓ   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r+   c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee   de	ej                  ej                  f   fd„Z
ˆ xZS )
ÚIJepaAttentionr   c                 ó(  •— t         ‰| �  «        || _        |j                  | _        t	        |d|j
                  |j                  z  «      | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  d¬«      | _        y )NÚhead_dimrx   F)ÚbiasT)r   r   r   Únum_attention_headsÚgetattrr&   r‡   Úattention_probs_dropout_probÚattention_dropoutru   Ú	is_causalr$   ÚLinearÚqkv_biasÚq_projÚk_projÚv_projÚo_proj©r(   r   r)   s     €r*   r   zIJepaAttention.__init__ª   s,  ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ!'×!DÑ!DˆÔØ—}‘} dÑ*ˆŒØˆŒä—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐeiÔjˆ�r+   NÚhidden_statesrt   rv   r-   c                 óš  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  sdn| j                  | j                  dœ|¤Ž\  }
} |
j                  g |¢d‘­Ž j!                  «       }
| j#                  |
«      }
|
|fS )NrO   r   r0   ç        )rJ   ru   )r1   r‡   r�   rZ   r4   r‘   r’   r   Úget_interfacer   Ú_attn_implementationrƒ   r{   rŒ   ru   rV   r€   r“   )r(   r•   rt   rv   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer‚   r�   s               r*   r5   zIJepaAttention.forward¸   sK  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆà˜LÐ(Ð(r+   ©N)r6   r7   r8   r   r   r:   r;   r   r   Útupler5   r<   r=   s   @r*   r…   r…   ©   sf   ø„ ðk˜{õ kð" /3ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð Ð+Ñ,ð	)ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷)r+   r…   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚIJepaMLPr   c                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y r    )r   r   r   r   Ú
hidden_actÚactivation_fnr$   rŽ   r&   Úintermediate_sizeÚfc1Úfc2r”   s     €r*   r   zIJepaMLP.__init__Û   sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r+   r•   r-   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r    )r¨   r¦   r©   )r(   r•   s     r*   r5   zIJepaMLP.forwardâ   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆàÐr+   ©	r6   r7   r8   r   r   r:   r;   r5   r<   r=   s   @r*   r£   r£   Ú   s,   ø„ ðK˜{õ Kð U§\¡\ð °e·l±l÷ r+   r£   c            	       óˆ   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee   dej                  fd„Z	ˆ xZ
S )
Ú
IJepaLayerr   c                 ój  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  ¬«      | _        t	        j
                  |j                  |j                  ¬«      | _	        t        |«      | _        t	        j                  |j                  «      | _        y )N©Úeps)r   r   r…   Ú	attentionr$   Ú	LayerNormr&   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterr£   ÚmlprH   rI   rJ   r”   s     €r*   r   zIJepaLayer.__init__ë   sz   ø€ Ü‰ÑÔÜ'¨Ó/ˆŒÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜ˜FÓ#ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r+   Nr•   rt   rv   r-   c                 óú   — |}| j                  |«      } | j                  ||fi |¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S r    )r´   r±   rJ   rµ   r¶   )r(   r•   rt   rv   Úresidualri   s         r*   r5   zIJepaLayer.forwardó   s�   € ð !ˆØ×-Ñ-¨mÓ<ˆØ)˜4Ÿ>™>¨-¸ÑRÈ6ÑRÑˆ�qØŸ™ ]Ó3ˆØ%¨Ñ0ˆð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆØ%¨Ñ0ˆàÐr+   r    )r6   r7   r8   r   r   r:   r;   r   r   r5   r<   r=   s   @r*   r­   r­   ê   sV   ø„ ð>˜{õ >ð /3ñà—|‘|ðð Ÿ™ tÑ+ðð Ð+Ñ,ð	ð
 
�‰÷r+   r­   c                   óŒ   ‡ — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZdZeedœZd	Z ej(                  «       ˆ fd
„«       Zˆ xZS )ÚIJepaPreTrainedModelr   Úijepar,   )ÚimageTr?   r­   )r•   Ú
attentionsrE   c                 ó$  •— t         ‰| �  |«       t        |t        j                  t        j
                  f«      rct        j                  |j                  d| j                  j                  ¬«       |j                  � t        j                  |j                  «       yyt        |t        «      rct        j                  |j                  d| j                  j                  ¬«       |j                  � t        j                  |j                  «       yyy)zInitialize the weightsr—   )ÚmeanÚstdN)r   Ú_init_weightsr!   r$   rŽ   r%   ÚinitÚtrunc_normal_Úweightr   Úinitializer_rangerˆ   Úzeros_r?   rG   rD   )r(   rp   r)   s     €r*   rÁ   z"IJepaPreTrainedModel._init_weights  s¼   ø€ ô 	‰Ñ˜fÔ%Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}°3¸D¿K¹K×<YÑ<YÕZØ�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜¤Ô0Ü×Ñ˜v×9Ñ9ÀÈÏÉ×IfÑIfÕgØ× Ñ Ð,Ü—‘˜F×-Ñ-Õ.ð -ð 1r+   )r6   r7   r8   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr­   r…   Ú_can_record_outputsÚ_input_embed_layerr:   Úno_gradrÁ   r<   r=   s   @r*   rº   rº   
  sx   ø… àÓØÐØ$€OØ!ÐØ&*Ð#Ø*¨LÐ9ÐØ€NØÐØÐØ"&ÐØ!Ðà#Ø$ñÐð ,Ðà€U‡]�]ƒ_ó
/ó ô
/r+   rº   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚIJepaPoolerr   c                 ó°   •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     | _	        y r    )
r   r   r$   rŽ   r&   Úpooler_output_sizeÚdenser   Ú
pooler_actÚ
activationr”   s     €r*   r   zIJepaPooler.__init__,  s>   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3LÑ3LÓMˆŒ
Ü  ×!2Ñ!2Ñ3ˆ�r+   r•   r-   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rÙ   rÛ   )r(   r•   Úfirst_token_tensorÚpooled_outputs       r*   r5   zIJepaPooler.forward1  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr+   r«   r=   s   @r*   rÖ   rÖ   +  s*   ø„ ð4˜{õ 4ð
 U§\¡\ð °e·l±l÷ r+   rÖ   c                   óÞ   ‡ — e Zd Zddededefˆ fd„Ze ed¬«      e	 	 	 	 dde	j                  dz  d	e	j                  dz  d
edz  de	j                  dz  dee   defd„«       «       «       Zˆ xZS )Ú
IJepaModelFr   Úadd_pooling_layerr@   c                 ó�  •— t         ‰| �  |«       || _        t        ||¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _
        t        j                  |j                  |j                  ¬«      | _        |rt        |«      nd| _        | j#                  «        yc c}w )zû
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )r@   r¯   N)r   r   r   r?   rK   r$   Ú
ModuleListÚrangeÚnum_hidden_layersr­   Úlayersr²   r&   r³   Ú	layernormrÖ   ÚpoolerÚ	post_init)r(   r   rá   r@   ri   r)   s        €r*   r   zIJepaModel.__init__<  s�   ø€ ô 	‰Ñ˜Ô ØˆŒÜ)¨&ÀÔPˆŒÜ—m‘mÄÀv×G_ÑG_ÓA`Ö$a¸A¤Z°Õ%7Ò$aÓbˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÙ->”k &Ô)ÀDˆŒà�‰Õùò	 %bs   ÁC)Útie_last_hidden_statesNr,   rb   ra   rt   rv   r-   c                 ó¶  — | j                   j                  j                  j                  j                  }|j                  |k7  r|j                  |«      }| j                  |||¬«      }t        | j                  ||¬«      }|}| j                  D ]  }	 |	||fi |¤Ž}Œ | j                  |«      }
| j                  �| j                  |
«      nd}t        |
|¬«      S )zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )rb   ra   )r   Úinputs_embedsrt   N)Úlast_hidden_stateÚpooler_output)rK   rE   r'   rÄ   ry   r   r   r   ræ   rç   rè   r
   )r(   r,   rb   ra   rt   rv   Úexpected_dtypeÚembedding_outputr•   ÚlayerÚsequence_outputrÞ   s               r*   r5   zIJepaModel.forwardL  sÛ   € ð  Ÿ™×9Ñ9×DÑD×KÑK×QÑQˆØ×Ñ Ò/Ø'Ÿ?™?¨>Ó:ˆLàŸ?™?Ø¨/ÐTlð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð
 )ˆØ—[‘[ò 	KˆEÙ! -°ÑJÀ6ÑJ‰Mð	Kð Ÿ.™.¨Ó7ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆä)¸OÐ[hÔiÐir+   )FF)NNNN)r6   r7   r8   r   rm   r   r   r   r   r:   r;   ro   r   r   r
   r5   r<   r=   s   @r*   rà   rà   :  s¾   ø„ ñ˜{ð ¸tð Ð]aõ ð   Ù¨EÔ2Øð -1Ø37Ø04Ø.2ñ jà—l‘l TÑ)ð jð ×)Ñ)¨DÑ0ð jð #'¨¡+ð	 jð
 Ÿ™ tÑ+ð jð Ð+Ñ,ð jð 
$ò jó ó 3ó  ô jr+   rà   aÒ  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )Úcustom_introc                   óœ   ‡ — e Zd Zdefˆ fd„Zee	 	 	 d
dej                  dz  dej                  dz  de	dz  de
e   def
d	„«       «       Zˆ xZS )ÚIJepaForImageClassificationr   c                 ó.  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j                  dkD  r*t        j                  |j                  |j                  «      nt        j                  «       | _	        | j                  «        y )NF)rá   r   )r   r   Ú
num_labelsrà   r»   r$   rŽ   r&   ÚIdentityÚ
classifierré   r”   s     €r*   r   z$IJepaForImageClassification.__init__�  ss   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ ¸%Ô@ˆŒ
ð OU×N_ÑN_ÐbcÒNcœ"Ÿ)™) F×$6Ñ$6¸×8IÑ8IÔJÔik×itÑitÓivˆŒð 	�‰Õr+   Nr,   Úlabelsra   rv   r-   c                 ó  —  | j                   |fd|i|¤Ž}|j                  }| j                  |j                  d¬«      «      }d}|� | j                  ||| j
                  fi |¤Ž}t        |||j                  |j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ra   r   )r]   N)ÚlossÚlogitsr•   r½   )	r»   rí   rù   r¿   Úloss_functionr   r   r•   r½   )	r(   r,   rú   ra   rv   Úoutputsrò   rý   rü   s	            r*   r5   z#IJepaForImageClassification.forward�  s¡   € ð  /9¨d¯j©jØñ/
à%=ð/
ð ñ/
ˆð
 "×3Ñ3ˆØ—‘ ×!5Ñ!5¸!Ð!5Ó!<Ó=ˆàˆØÐØ%�4×%Ñ% f¨f°d·k±kÑLÀVÑLˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )NNN)r6   r7   r8   r   r   r   r   r:   r;   rm   r   r   r   r5   r<   r=   s   @r*   rõ   rõ   r  s‚   ø„ ð
˜{õ 
ð Øð -1Ø&*Ø04ñ	
à—l‘l TÑ)ð
ð —‘˜tÑ#ð
ð #'¨¡+ð	
ð
 Ð+Ñ,ð
ð 
ò
ó ó ô
r+   rõ   )rº   rà   rõ   )Nr—   )0Úcollections.abcr   r   r:   Útorch.nnr$   Ú r   rÂ   Úactivationsr   Úmasking_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_ijepar   ÚModuler   r?   r;   Úfloatrƒ   r…   r£   r­   rº   rÖ   rà   rõ   Ú__all__© r+   r*   ú<module>r     s  ð÷ /ã Ý å &Ý !Ý 6Ý 9ß Qß FÝ &ß BÑ Bß IÝ 5Ý ,ôH˜2Ÿ9™9ô Hô<S�b—i‘iô Sðx !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8.)�R—Y‘Yô .)ôbˆr�y‰yô ô Ð+ô ð@ ô/˜?ó /ó ð/ô@�"—)‘)ô ð ô4jÐ%ó 4jó ð4jñn ðôô.
Ð"6ó .
óð.
òb P�r+   