Ë
    ÛÓ^j¨a  ã                   óì  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%  ejL                  e'«      Z( G d„ dejR                  «      Z* G d„ dejR                  «      Z+	 	 d=dejR                  dejX                  dejX                  dejX                  dejX                  dz  de-dz  de-dee   fd„Z. G d„ d ejR                  «      Z/ G d!„ d"ejR                  «      Z0 G d#„ d$ejR                  «      Z1 G d%„ d&ejR                  «      Z2 G d'„ d(ejR                  «      Z3 G d)„ d*ejR                  «      Z4 G d+„ d,ejR                  «      Z5 G d-„ d.e«      Z6e G d/„ d0e«      «       Z7 G d1„ d2e7«      Z8e G d3„ d4e7«      «       Z9 ed5¬6«       G d7„ d8e7«      «       Z: ed9¬6«       G d:„ d;ee7«      «       Z;g d<¢Z<y)>zPyTorch DINOv2 model.é    N)ÚCallable)Únné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚGradientCheckpointingLayer)ÚBackboneOutputÚBaseModelOutputÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚDinov2Configc                   óÄ   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dededej                  fd	„Z	dd
ej                  dej                  dz  dej                  fd„Z
ˆ xZS )ÚDinov2EmbeddingszM
    Construct the CLS token, mask token, position and patch embeddings.
    ÚconfigÚreturnNc                 óz  •— t         ‰| �  «        t        j                  t	        j
                  dd|j                  «      «      | _        |j                  r8t        j                  t	        j                  d|j                  «      «      | _
        t        |«      | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                  «      «      | _        t        j                  |j                   «      | _        |j$                  | _        |j                  | _        || _        y )Nr   )ÚsuperÚ__init__r   Ú	ParameterÚtorchÚrandnÚhidden_sizeÚ	cls_tokenÚuse_mask_tokenÚzerosÚ
mask_tokenÚDinov2PatchEmbeddingsÚpatch_embeddingsÚnum_patchesÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutÚ
patch_sizer   )Úselfr   r,   Ú	__class__s      €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dinov2/modeling_dinov2.pyr!   zDinov2Embeddings.__init__+   sÚ   ø€ Ü‰ÑÔäŸ™¤e§k¡k°!°Q¸×8JÑ8JÓ&KÓLˆŒØ× Ò Ü Ÿl™l¬5¯;©;°q¸&×:LÑ:LÓ+MÓNˆDŒOÜ 5°fÓ =ˆÔØ×+Ñ+×7Ñ7ˆÜ#%§<¡<´·±¸A¸{ÈQ¹ÐPV×PbÑPbÓ0cÓ#dˆÔ Ü—z‘z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØ$×3Ñ3ˆÔØˆ�ó    Ú
embeddingsÚheightÚwidthc                 ó  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }|j                  }t        j                  j                  |j                  t        j                  «      |	|
fdd	¬
«      j                  |¬«      }|j                  dddd«      j                  dd|«      }t        j                   ||fd¬«      S )a-  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and interpolation at torch.float32 precision.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Néÿÿÿÿg      à?r   r   é   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údtype©Údim)Úshaper-   r#   ÚjitÚ
is_tracingr1   r   ÚreshapeÚpermuterA   r   Ú
functionalÚinterpolateÚtoÚfloat32ÚviewÚcat)r2   r6   r7   r8   r,   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrC   Ú
new_heightÚ	new_widthÚsqrt_num_positionsÚtarget_dtypes                r4   Úinterpolate_pos_encodingz)Dinov2Embeddings.interpolate_pos_encoding9   s‹  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×2Ñ2²1°b°q°b°5Ñ9ˆØ×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆØ&×,Ñ,ˆÜŸ-™-×3Ñ3Ø×ÑœuŸ}™}Ó-Ø˜iÐ(ØØð	 4ó 
÷
 ‰"�<ˆ"Ó
 ð 	ð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr5   Úpixel_valuesÚbool_masked_posc                 óD  — |j                   \  }}}}| j                  j                  j                  j                  }| j                  |j                  |¬«      «      }|�d| j                  rXt        j                  |j                  d«      | j                  j                  |j                  «      j                  d«      |«      }| j                  j                  |dd«      }	t        j                  |	|fd¬«      }|| j                  |||«      z   }| j                  |«      }|S )Nr@   r:   r   r   rB   )rD   r+   Ú
projectionÚweightrA   rK   r'   r#   ÚwhereÚ	unsqueezer)   r&   ÚexpandrN   rV   r0   )
r2   rW   rX   Ú
batch_sizeÚ_r7   r8   rU   r6   Ú
cls_tokenss
             r4   ÚforwardzDinov2Embeddings.forwarda   sý   € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uØ×,Ñ,×7Ñ7×>Ñ>×DÑDˆØ×*Ñ*¨<¯?©?À¨?Ó+NÓOˆ
àÐ&¨4×+>Ò+>ÜŸ™Ø×)Ñ)¨"Ó-¨t¯©×/AÑ/AÀ*×BRÑBRÓ/S×/]Ñ/]Ð^_Ó/`ÐblóˆJð
 —^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ü—Y‘Y 
¨JÐ7¸QÔ?ˆ
ð   $×"?Ñ"?À
ÈFÐTYÓ"ZÑZˆ
à—\‘\ *Ó-ˆ
àÐr5   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r!   r#   ÚTensorÚintrV   rb   Ú__classcell__©r3   s   @r4   r   r   &   s}   ø„ ñð˜|ð °õ ð&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]ió &DñP E§L¡Lð À5Ç<Á<ÐRVÑCVð Ðbg×bnÑbn÷ r5   r   c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )r*   zì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 óÌ  •— t         ‰| �  «        |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                  «      r|n||f}t        |t        j                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||¬«      | _        y )Nr   r   )Úkernel_sizeÚstride)r    r!   Ú
image_sizer1   Únum_channelsr%   Ú
isinstanceÚcollectionsÚabcÚIterabler,   r   ÚConv2drZ   )r2   r   rp   r1   rq   r%   r,   r3   s          €r4   r!   zDinov2PatchEmbeddings.__init__~   sÔ   ø€ Ü‰ÑÔØ!'×!2Ñ!2°F×4EÑ4E�Jˆ
Ø$*×$7Ñ$7¸×9KÑ9K�kˆä#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔäŸ)™) L°+È:Ð^hÔiˆ�r5   rW   r   c                 óÚ   — |j                   d   }|| j                  k7  rt        d| j                  › d|› d�«      ‚| j                  |«      j	                  d«      j                  dd«      }|S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.r;   )rD   rq   Ú
ValueErrorrZ   ÚflattenÚ	transpose)r2   rW   rq   r6   s       r4   rb   zDinov2PatchEmbeddings.forward�   sz   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜðØ!×.Ñ.Ð/¨y¸¸ÀaðIóð ð —_‘_ \Ó2×:Ñ:¸1Ó=×GÑGÈÈ1ÓMˆ
ØÐr5   )	rd   re   rf   rg   r!   r#   rh   rb   rj   rk   s   @r4   r*   r*   w   s)   ø„ ñôjð E§L¡Lð °U·\±\÷ r5   r*   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr0   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr:   ç      à¿r;   r   rB   )ÚpÚtrainingr   )
r=   r#   Úmatmulr{   r   rI   Úsoftmaxr0   r†   Ú
contiguous)
r|   r}   r~   r   r€   r�   r0   r‚   Úattn_weightsÚattn_outputs
             r4   Úeager_attention_forwardrŒ   ™   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r5   c                   ó„   ‡ — e Zd Zdefˆ fd„Zdej                  dee   de	ej                  ej                  f   fd„Z
ˆ xZS )ÚDinov2SelfAttentionr   c                 ó2  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                   ¬«      | _        t        j                  |j                  | j                  |j                   ¬«      | _        t        j                  |j                  | j                  |j                   ¬«      | _        y )	Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads rx   r„   F©Úbias)r    r!   r%   Únum_attention_headsÚhasattrry   r   ri   Úattention_head_sizeÚall_head_sizeÚattention_probs_dropout_probÚdropout_probr�   Ú	is_causalr   ÚLinearÚqkv_biasr}   r~   r   ©r2   r   r3   s     €r4   r!   zDinov2SelfAttention.__init__·   sF  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ" 6×#5Ñ#5Ð"6ð 7Ø×3Ñ3Ð4°Að7óð ð
 ˆŒØ#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ"×?Ñ?ˆÔØ×/Ñ/°Ñ5ˆŒØˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÈ&Ï/É/ÔZˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆ�
r5   Úhidden_statesr‚   r   c                 ó°  — |j                   d   }|d| j                  | j                  f} | j                  |«      j                  |Ž j                  dd«      } | j                  |«      j                  |Ž j                  dd«      } | j                  |«      j                  |Ž j                  dd«      }t        j                  | j                  j                  t        «      } || |||d f| j                  | j                  | j                  sdn| j                   dœ|¤Ž\  }	}
|	j#                  «       d d | j$                  fz   }|	j'                  |«      }	|	|
fS )Nr   r:   r   r;   ç        )r™   r�   r0   éþÿÿÿ)rD   r“   r•   r~   rM   r{   r   r}   r   Úget_interfacer   Ú_attn_implementationrŒ   r™   r�   r†   r˜   r=   r–   rG   )r2   r�   r‚   r_   Ú	new_shapeÚ	key_layerÚvalue_layerÚquery_layerÚattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes               r4   rb   zDinov2SelfAttention.forwardË   sY  € ð
 #×(Ñ(¨Ñ+ˆ
Ø  D×$<Ñ$<¸d×>VÑ>VÐVˆ	à0�D—H‘H˜]Ó+×0Ñ0°)Ð<×FÑFÀqÈ!ÓLˆ	Ø4�d—j‘j Ó/×4Ñ4°iÐ@×JÑJÈ1ÈaÓPˆØ4�d—j‘j Ó/×4Ñ4°iÐ@×JÑJÈ1ÈaÓPˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ *=ØØØØØð
*
ð —n‘nØ—L‘LØ#Ÿ}š}‘C°$×2CÑ2Cñ
*
ð ñ
*
Ñ&ˆ�ð #0×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×-Ñ-Ð.EÓFˆà˜oÐ-Ð-r5   )rd   re   rf   r   r!   r#   rh   r   r   Útuplerb   rj   rk   s   @r4   rŽ   rŽ   ¶   sN   ø„ ð]˜|õ ]ð(.à—|‘|ð.ð Ð+Ñ,ð.ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	.r5   rŽ   c                   óx   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  fd„Zˆ xZ	S )ÚDinov2SelfOutputz£
    The residual connection is defined in Dinov2Layer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r   c                 óÈ   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        y rc   )	r    r!   r   rš   r%   Údenser.   r/   r0   rœ   s     €r4   r!   zDinov2SelfOutput.__init__ô   sB   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r5   r�   Úinput_tensorr   c                 óJ   — | j                  |«      }| j                  |«      }|S rc   )r¯   r0   )r2   r�   r°   s      r4   rb   zDinov2SelfOutput.forwardù   s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr5   ©
rd   re   rf   rg   r   r!   r#   rh   rb   rj   rk   s   @r4   r­   r­   î   s=   ø„ ñð
>˜|õ >ð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r5   r­   c                   óf   ‡ — e Zd Zdefˆ fd„Zdej                  dee   dej                  fd„Z	ˆ xZ
S )ÚDinov2Attentionr   c                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y rc   )r    r!   rŽ   Ú	attentionr­   Úoutputrœ   s     €r4   r!   zDinov2Attention.__init__  s&   ø€ Ü‰ÑÔÜ,¨VÓ4ˆŒÜ& vÓ.ˆ�r5   r�   r‚   r   c                 óV   —  | j                   |fi |¤Ž\  }}| j                  ||«      }|S rc   )r¶   r·   )r2   r�   r‚   Úself_attn_outputr`   r·   s         r4   rb   zDinov2Attention.forward  s5   € ð
 -˜dŸn™n¨]ÑE¸fÑEÑÐ˜!Ø—‘Ð-¨}Ó=ˆØˆr5   )rd   re   rf   r   r!   r#   rh   r   r   rb   rj   rk   s   @r4   r´   r´      s>   ø„ ð/˜|õ /ð
à—|‘|ðð Ð+Ñ,ðð 
�‰÷	r5   r´   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚDinov2LayerScaler   c                 óª   •— t         ‰| �  «        t        j                  |j                  t        j                  |j                  «      z  «      | _        y rc   )	r    r!   r   r"   Úlayerscale_valuer#   Úonesr%   Úlambda1rœ   s     €r4   r!   zDinov2LayerScale.__init__  s8   ø€ Ü‰ÑÔÜ—|‘| F×$;Ñ$;¼e¿j¹jÈ×I[ÑI[Ó>\Ñ$\Ó]ˆ�r5   Úhidden_statec                 ó    — || j                   z  S rc   )r¿   ©r2   rÀ   s     r4   rb   zDinov2LayerScale.forward  s   € Ø˜dŸl™lÑ*Ð*r5   ©r   N©rd   re   rf   r!   r#   rh   rb   rj   rk   s   @r4   r»   r»     s$   ø„ õ^ð+ E§L¡Lð +°U·\±\÷ +r5   r»   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú	Dinov2MLPr   c                 ó~  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        j                  ||d¬«      | _        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  ||d¬«      | _        y )NTr‘   )r    r!   r%   ri   Ú	mlp_ratior   rš   Úfc1rr   Ú
hidden_actÚstrr   Ú
activationÚfc2©r2   r   Úin_featuresÚout_featuresÚhidden_featuresr3   s        €r4   r!   zDinov2MLP.__init__  s�   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ—9‘9˜[¨/ÀÔEˆŒÜ�f×'Ñ'¬Ô-Ü$ V×%6Ñ%6Ñ7ˆD�Oà$×/Ñ/ˆDŒOÜ—9‘9˜_¨lÀÔFˆ�r5   rÀ   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rc   )rÉ   rÌ   rÍ   rÂ   s     r4   rb   zDinov2MLP.forward%  s2   € Ø—x‘x Ó-ˆØ—‘ |Ó4ˆØ—x‘x Ó-ˆØÐr5   rÃ   rÄ   rk   s   @r4   rÆ   rÆ     s$   ø„ õ	Gð E§L¡Lð °U·\±\÷ r5   rÆ   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚDinov2SwiGLUFFNr   c                 ó0  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        |dz  dz  «      dz   dz  dz  }t        j                  |d|z  d¬«      | _        t        j                  ||d¬«      | _        y )Nr;   r   é   é   Tr‘   )	r    r!   r%   ri   rÈ   r   rš   Ú
weights_inÚweights_outrÎ   s        €r4   r!   zDinov2SwiGLUFFN.__init__-  sŠ   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ˜°Ñ2°QÑ6Ó7¸!Ñ;ÀÑAÀAÑEˆäŸ)™) K°°_Ñ1DÈ4ÔPˆŒÜŸ9™9 _°lÈÔNˆÕr5   rÀ   c                 ó¶   — | j                  |«      }|j                  dd¬«      \  }}t        j                  j	                  |«      |z  }| j                  |«      S )Nr;   r:   rB   )rØ   Úchunkr   rI   ÚsilurÙ   )r2   rÀ   Úx1Úx2Úhiddens        r4   rb   zDinov2SwiGLUFFN.forward6  sS   € Ø—‘ |Ó4ˆØ×#Ñ# A¨2Ð#Ó.‰ˆˆBÜ—‘×#Ñ# BÓ'¨"Ñ,ˆØ×Ñ Ó'Ð'r5   rÃ   rÄ   rk   s   @r4   rÔ   rÔ   ,  s$   ø„ õOð( E§L¡Lð (°U·\±\÷ (r5   rÔ   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚDinov2DropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr   Nc                 ó0   •— t         ‰| �  «        || _        y rc   )r    r!   râ   )r2   râ   r3   s     €r4   r!   zDinov2DropPath.__init__E  s   ø€ Ü‰ÑÔØ"ˆ�r5   r�   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )NrŸ   r   r   )r   )rA   Údevice)
râ   r†   rD   Úndimr#   ÚrandrA   rå   ÚfloorÚdiv)r2   r�   Ú	keep_probrD   Úrandom_tensors        r4   rb   zDinov2DropPath.forwardI  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r5   c                 ó    — d| j                   › �S )Nzp=)râ   ©r2   s    r4   Ú
extra_reprzDinov2DropPath.extra_reprR  s   € Ø�D—N‘NÐ#Ð$Ð$r5   )rŸ   )rd   re   rf   rg   Úfloatr!   r#   rh   rb   rË   rî   rj   rk   s   @r4   rá   rá   >  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r5   rá   c                   ód   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚDinov2LayerzCThis corresponds to the Block class in the original implementation.r   r   Nc                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        |«      | _
        |j                  dkD  rt        |j                  «      nt        j                  «       | _        t        j                  |j                  |j
                  ¬«      | _        |j                   rt#        |«      | _        nt'        |«      | _        t        |«      | _        y )N©ÚepsrŸ   )r    r!   r   Ú	LayerNormr%   Úlayer_norm_epsÚnorm1r´   r¶   r»   Úlayer_scale1Údrop_path_raterá   ÚIdentityÚ	drop_pathÚnorm2Úuse_swiglu_ffnrÔ   ÚmlprÆ   Úlayer_scale2rœ   s     €r4   r!   zDinov2Layer.__init__Y  s½   ø€ Ü‰ÑÔä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
Ü(¨Ó0ˆŒÜ,¨VÓ4ˆÔØBH×BWÑBWÐZ]ÒB]œ¨×(=Ñ(=Ô>Ôce×cnÑcnÓcpˆŒä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
à× Ò Ü& vÓ.ˆD�Hä  Ó(ˆDŒHÜ,¨VÓ4ˆÕr5   r�   c                 ó"  — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }|S rc   )r÷   r¶   rø   rû   rü   rþ   rÿ   )r2   r�   Úhidden_states_normÚself_attention_outputÚlayer_outputs        r4   rb   zDinov2Layer.forwardi  s”   € ð "ŸZ™Z¨Ó6ÐØ $§¡Ð/AÓ BÐØ $× 1Ñ 1Ð2GÓ HÐð Ÿ™Ð'<Ó=ÀÑMˆð —z‘z -Ó0ˆØ—x‘x Ó-ˆØ×(Ñ(¨Ó6ˆð —~‘~ lÓ3°mÑCˆàÐr5   r²   rk   s   @r4   rñ   rñ   V  s8   ø„ ÙMð5˜|ð 5°õ 5ð à—|‘|ðð 
�‰÷r5   rñ   c                   óÔ   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZeedœZ ej$                  «       dej(                  ej*                  z  ej,                  z  d	d
fˆ fd„«       Zˆ xZS )ÚDinov2PreTrainedModelr   Údinov2rW   )ÚimageTrñ   )r�   Ú
attentionsr|   r   Nc                 ó.  •— t         ‰| �  |«       t        |t        j                  t        j
                  f«      rct        j                  |j                  d| j                  j                  ¬«       |j                  � t        j                  |j                  «       yyt        |t        «      r£t        j                  |j                  d| j                  j                  ¬«       t        j                  |j                  d| j                  j                  ¬«       | j                  j                   r t        j                  |j"                  «       yyt        |t$        «      r5t        j&                  |j(                  | j                  j*                  «       yy)zInitialize the weightsrŸ   )ÚmeanÚstdN)r    Ú_init_weightsrr   r   rš   rv   ÚinitÚtrunc_normal_r[   r   Úinitializer_ranger’   Úzeros_r   r-   r&   r'   r)   r»   Ú	constant_r¿   r½   )r2   r|   r3   s     €r4   r  z#Dinov2PreTrainedModel._init_weights�  s  ø€ ô 	‰Ñ˜fÔ%Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}°3¸D¿K¹K×<YÑ<YÕZØ�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜Ô 0Ô1Ü×Ñ˜v×9Ñ9ÀÈÏÉ×IfÑIfÕgÜ×Ñ˜v×/Ñ/°c¸t¿{¹{×?\Ñ?\Õ]Ø�{‰{×)Ò)Ü—‘˜F×-Ñ-Õ.ð *ä˜Ô 0Ô1Ü�N‰N˜6Ÿ>™>¨4¯;©;×+GÑ+GÕHð 2r5   )rd   re   rf   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendrñ   rŽ   Ú_can_record_outputsr#   Úno_gradr   rš   rv   rõ   r  rj   rk   s   @r4   r  r    s‘   ø… àÓØ ÐØ$€OØ!ÐØ&*Ð#Ø&˜ÐØ€NØÐØÐØ"&Ðà$Ø)ñÐð
 €U‡]�]ƒ_ðI B§I¡I°·	±	Ñ$9¸B¿L¹LÑ$Hð IÈTô Ió ôIr5   r  c                   ót   ‡ — e Zd Zdefˆ fd„Ze ed¬«      dej                  de	e
   defd„«       «       Zˆ xZS )	ÚDinov2Encoderr   c                 óÖ   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        | j                  «        y c c}w rc   )	r    r!   r   Ú
ModuleListÚrangeÚnum_hidden_layersrñ   ÚlayerÚ	post_init©r2   r   r`   r3   s      €r4   r!   zDinov2Encoder.__init__¢  sK   ø€ Ü‰Ñ˜Ô Ü—]‘]ÄÀv×G_ÑG_ÓA`Ö#a¸A¤K°Õ$7Ò#aÓbˆŒ
Ø�‰Õùò $bs   ·A&F)Útie_last_hidden_statesr�   r‚   r   c                 óL   — | j                   D ]
  } ||«      }Œ t        |¬«      S )N)Úlast_hidden_state)r$  r   )r2   r�   r‚   Úlayer_modules       r4   rb   zDinov2Encoder.forward§  s.   € ð !ŸJ™Jò 	8ˆLÙ(¨Ó7‰Mð	8ô °Ô?Ð?r5   )rd   re   rf   r   r!   r   r   r#   rh   r   r   r   rb   rj   rk   s   @r4   r  r  ¡  sU   ø„ ð˜|õ ð
  Ù¨EÔ2ð@ U§\¡\ð @¸VÐDVÑ=Wð @Ð\kò @ó 3ó  ô@r5   r  c                   óœ   ‡ — e Zd Zdefˆ fd„Zdefd„Zee	 	 d
de	j                  dz  de	j                  dz  dee   defd	„«       «       Zˆ xZS )ÚDinov2Modelr   c                 óò   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        | j                  «        y )Nró   )r    r!   r   r   r6   r  Úencoderr   rõ   r%   rö   Ú	layernormr%  rœ   s     €r4   r!   zDinov2Model.__init__²  sY   ø€ Ü‰Ñ˜Ô ØˆŒä*¨6Ó2ˆŒÜ$ VÓ,ˆŒäŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒð 	�‰Õr5   r   c                 ó.   — | j                   j                  S rc   ©r6   r+   rí   s    r4   Úget_input_embeddingsz Dinov2Model.get_input_embeddings¾  ó   € Ø�‰×/Ñ/Ð/r5   NrW   rX   r‚   c                 ó   — |€t        d«      ‚| j                  ||¬«      } | j                  |fi |¤Ž}|j                  }| j	                  |«      }|dd…ddd…f   }t        |||j                  |j                  ¬«      S )zï
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for
            pre-training.
        Nz You have to specify pixel_values)rX   r   )r)  Úpooler_outputr�   r  )ry   r6   r.  r)  r/  r   r�   r  )r2   rW   rX   r‚   Úembedding_outputÚencoder_outputsÚsequence_outputÚpooled_outputs           r4   rb   zDinov2Model.forwardÁ  s�   € ð ÐÜÐ?Ó@Ð@àŸ?™?¨<È˜?ÓYÐà+7¨4¯<©<Ð8HÑ+SÈFÑ+SˆØ)×;Ñ;ˆØŸ.™.¨Ó9ˆØ'ª¨1ªa¨Ñ0ˆä)Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r5   ©NN)rd   re   rf   r   r!   r*   r2  r   r   r#   rh   r   r   r   rb   rj   rk   s   @r4   r,  r,  °  s~   ø„ ð
˜|õ 
ð0Ð&;ó 0ð Øð -1Ø/3ñ
à—l‘l TÑ)ð
ð Ÿ™¨Ñ,ð
ð Ð+Ñ,ð	
ð
 
$ò
ó ó ô
r5   r,  z§
    Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.
    )Úcustom_introc                   ó”   ‡ — e Zd Zdeddfˆ fd„Zee	 	 d	dej                  dz  dej                  dz  de	e
   defd„«       «       Zˆ xZS )
ÚDinov2ForImageClassificationr   r   Nc                 ó0  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j                  dkD  r-t        j                  |j                  dz  |j                  «      nt        j                  «       | _	        | j                  «        y )Nr   r;   )r    r!   Ú
num_labelsr,  r  r   rš   r%   rú   Ú
classifierr%  rœ   s     €r4   r!   z%Dinov2ForImageClassification.__init__ç  sy   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ! &Ó)ˆŒð EK×DUÑDUÐXYÒDYŒB�I‰I�f×(Ñ(¨1Ñ,¨f×.?Ñ.?Ô@Ô_a×_jÑ_jÓ_lð 	Œð
 	�‰Õr5   rW   Úlabelsr‚   c                 óh  —  | j                   |fi |¤Ž}|j                  }|dd…df   }|dd…dd…f   }t        j                  ||j	                  d¬«      gd¬«      }| j                  |«      }	d}
|� | j                  ||	| j                  fi |¤Ž}
t        |
|	|j                  |j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r   rB   )ÚlossÚlogitsr�   r  )r  r)  r#   rN   r
  r@  Úloss_functionr   r   r�   r  )r2   rW   rA  r‚   Úoutputsr8  r&   Úpatch_tokensÚlinear_inputrD  rC  s              r4   rb   z$Dinov2ForImageClassification.forwardõ  sÂ   € ð /:¨d¯k©k¸,Ñ.QÈ&Ñ.Qˆà!×3Ñ3ˆØ#¢A q DÑ)ˆ	Ø&¢q¨!©" uÑ-ˆä—y‘y )¨\×->Ñ->À1Ð->Ó-EÐ!FÈAÔNˆØ—‘ Ó.ˆàˆØÐØ%�4×%Ñ% f¨f°d·k±kÑLÀVÑLˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r5   r:  )rd   re   rf   r   r!   r   r   r#   rh   r   r   r   rb   rj   rk   s   @r4   r=  r=  à  sx   ø„ ð˜|ð °õ ð Øð -1Ø&*ñ
à—l‘l TÑ)ð
ð —‘˜tÑ#ð
ð Ð+Ñ,ð	
ð
 
ò
ó ó ô
r5   r=  zO
    Dinov2 backbone, to be used with frameworks like DETR and MaskFormer.
    c            	       óv   ‡ — e Zd Zˆ fd„Zdefd„Zeeede	j                  dee   defd„«       «       «       Zˆ xZS )ÚDinov2Backbonec                 óX  •— t         ‰| �  |«       t        |j                  dz   «      D �cg c]  }|j                  ‘Œ c}| _        t        |«      | _        t        |«      | _	        t        j                  |j                  |j                  ¬«      | _        | j                  «        y c c}w )Nr   ró   )r    r!   r"  r#  r%   Únum_featuresr   r6   r  r.  r   rõ   rö   r/  r%  r&  s      €r4   r!   zDinov2Backbone.__init__  s‚   ø€ Ü‰Ñ˜Ô ä9>¸v×?WÑ?WÐZ[Ñ?[Ó9\Ö]°A˜V×/Ó/Ò]ˆÔÜ*¨6Ó2ˆŒÜ$ VÓ,ˆŒäŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒð 	�‰Õùò ^s   «B'r   c                 ó.   — | j                   j                  S rc   r1  rí   s    r4   r2  z#Dinov2Backbone.get_input_embeddings*  r3  r5   rW   r‚   c                 ó„  — d|d<   | j                  |«      } | j                  |fi |¤Ž}|j                  }g }t        | j                  |«      D ]Ð  \  }}|| j
                  v sŒ| j                  j                  r| j                  |«      }| j                  j                  rn|dd…dd…f   }|j                  \  }	}
}}| j                  j                  }|j                  |	||z  ||z  d«      }|j                  dddd«      j                  «       }|j                  |«       ŒÒ t!        t#        |«      ||j$                  ¬	«      S )
av  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
        >>> model = AutoBackbone.from_pretrained(
        ...     "facebook/dinov2-base", out_features=["stage2", "stage5", "stage8", "stage11"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 16, 16]
        ```TÚoutput_hidden_statesNr   r:   r   r   r;   )Úfeature_mapsr�   r  )r6   r.  r�   ÚzipÚstage_namesrÐ   r   Úapply_layernormr/  Úreshape_hidden_statesrD   r1   rG   rH   r‰   Úappendr   r«   r  )r2   rW   r‚   r6  r·   r�   rP  ÚstagerÀ   r_   r`   r7   r8   r1   s                 r4   rb   zDinov2Backbone.forward-  sE  € ðD *.ˆÐ%Ñ&àŸ?™?¨<Ó8ÐØ". $§,¡,Ð/?Ñ"JÀ6Ñ"JˆØ×,Ñ,ˆàˆÜ#& t×'7Ñ'7¸Ó#Gò 	2ÑˆE�<Ø˜×)Ñ)Ò)Ø—;‘;×.Ò.Ø#'§>¡>°,Ó#?�LØ—;‘;×4Ò4Ø#/²°1±2°Ñ#6�Lð 4@×3EÑ3EÑ0�J  6¨5Ø!%§¡×!7Ñ!7�JØ#/×#7Ñ#7¸
ÀFÈjÑDXÐZ_ÐcmÑZmÐoqÓ#r�LØ#/×#7Ñ#7¸¸1¸aÀÓ#C×#NÑ#NÓ#P�LØ×#Ñ# LÕ1ð	2ô Ü˜|Ó,Ø'Ø×(Ñ(ô
ð 	
r5   )rd   re   rf   r!   r*   r2  r   r	   r   r#   rh   r   r   r   rb   rj   rk   s   @r4   rJ  rJ    s_   ø„ ô
ð0Ð&;ó 0ð Ø Øð8
à—l‘lð8
ð Ð+Ñ,ð8
ð 
ò	8
ó ó !ó ô8
r5   rJ  )r=  r,  r  rJ  )NrŸ   )=rg   Úcollections.abcrs   r   r#   r   Ú r   r  Úactivationsr   Úbackbone_utilsr   r	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_dinov2r   Ú
get_loggerrd   ÚloggerÚModuler   r*   rh   rï   rŒ   rŽ   r­   r´   r»   rÆ   rÔ   rá   rñ   r  r  r,  r=  rJ  Ú__all__© r5   r4   ú<module>rh     s  ðñ ã Ý $ã Ý å &Ý !ß HÝ 9ß rÓ rß FÝ &ß KÓ Kß IÝ 5Ý .ð 
ˆ×	Ñ	˜HÓ	%€ôN�r—y‘yô Nôb˜BŸI™Iô ðP !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô:4.˜"Ÿ)™)ô 4.ôp�r—y‘yô ô$�b—i‘iô ô +�r—y‘yô +ô�—	‘	ô ô&(�b—i‘iô (ô$%�R—Y‘Yô %ô0&Ð,ô &ðR ôI˜Oó Ió ðIôB@Ð)ô @ð ô,
Ð'ó ,
ó ð,
ñ^ ðôô/
Ð#8ó /
óð/
ñd ðôô
K
�]Ð$9ó K
óð
K
ò\ e�r5   