Ë
    ÛÓ^j 4  ã                   óx  — d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZmZmZmZ ddlmZ  G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z  ed¬«      e G d„ de«      «       «       Z! ed¬«       G d„ de«      «       Z"g d ¢Z#y)!z7PyTorch DeiT (Data-efficient Image Transformers) model.é    )Ú	dataclassN)Únné   )Úinitialization)ÚBaseModelOutputWithPoolingÚMaskedImageModelingOutput)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleé   )ÚViTEmbeddingsÚViTForImageClassificationÚViTForMaskedImageModelingÚViTModelÚViTPreTrainedModelé   )Ú
DeiTConfigc            	       óÒ   ‡ — e Zd ZdZddededdfˆ fd„Zdej                  de	d	e	dej                  fd
„Z
	 	 ddej                  dej                  dz  dedej                  fd„Zˆ xZS )ÚDeiTEmbeddingsaÇ  
    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.

    Differences from ViTEmbeddings:
    - Adds a distillation token (for distillation pre-training).
    - Position embeddings include +2 slots (CLS + distillation) instead of +1.
    - interpolate_pos_encoding handles 2 special tokens instead of 1.
    - forward concatenates distillation token and handles position encoding for both.
    ÚconfigÚuse_mask_tokenÚreturnNc                 ó°  •— t         ‰| �  ||¬«       t        j                  t	        j
                  dd|j                  «      «      | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                  «      «      | _
        t        j                  t	        j
                  dd|j                  «      «      | _        y )N)r   r   r   )ÚsuperÚ__init__r   Ú	ParameterÚtorchÚzerosÚhidden_sizeÚ	cls_tokenÚpatch_embeddingsÚnum_patchesÚposition_embeddingsÚdistillation_token)Úselfr   r   r%   Ú	__class__s       €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/deit/modular_deit.pyr   zDeiTEmbeddings.__init__2   s–   ø€ Ü‰Ñ˜°ÐÔ?ÜŸ™¤e§k¡k°!°Q¸×8JÑ8JÓ&KÓLˆŒØ×+Ñ+×7Ñ7ˆä#%§<¡<´·±¸A¸{ÈQ¹ÐPV×PbÑPbÓ0cÓ#dˆÔ Ü"$§,¡,¬u¯{©{¸1¸aÀ×ASÑASÓ/TÓ"UˆÕó    Ú
embeddingsÚheightÚwidthc                 ó¦  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }t        j                  j                  ||	|
fdd	¬
«      }|j                  dddd«      j                  dd|«      }t        j                  ||fd¬«      S )a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and 2 class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Néÿÿÿÿç      à?r   r   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)Úshaper&   r    ÚjitÚ
is_tracingÚ
patch_sizer   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r(   r,   r-   r.   r%   Únum_positionsÚclass_and_dist_pos_embedÚpatch_pos_embedr7   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r*   Úinterpolate_pos_encodingz'DeiTEmbeddings.interpolate_pos_encoding:   sb  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à#'×#;Ñ#;ºA¸rÀ¸r¸EÑ#BÐ Ø×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰yÐ2°OÐDÈ!ÔLÐLr+   Úpixel_valuesÚbool_masked_posrH   c                 óÒ  — |j                   \  }}}}| j                  |«      }|j                  «       \  }}	}|�K| j                  j	                  ||	d«      }
|j                  d«      j                  |
«      }|d|z
  z  |
|z  z   }| j                  j	                  |dd«      }| j                  j	                  |dd«      }t        j                  |||fd¬«      }|r|| j                  |||«      z   }ne|| j                  d   k7  s|| j                  d   k7  r2t        d|› d|› d| j                  d   › d| j                  d   › d	�	«      ‚|| j                  z   }| j                  |«      }|S )
Nr0   g      ð?r   r6   r   zInput image size (Ú*z) doesn't match model (z).)r8   r$   r3   Ú
mask_tokenÚexpandÚ	unsqueezeÚtype_asr#   r'   r    rA   rH   Ú
image_sizeÚ
ValueErrorr&   Údropout)r(   rI   rJ   rH   Ú_r-   r.   r,   Ú
batch_sizeÚ
seq_lengthÚmask_tokensÚmaskÚ
cls_tokensÚdistillation_tokenss                 r*   ÚforwardzDeiTEmbeddings.forwardb   sƒ  € ð +×0Ñ0Ñˆˆ1ˆf�eØ×*Ñ*¨<Ó8ˆ
à$.§O¡OÓ$5Ñ!ˆ
�J àÐ&ØŸ/™/×0Ñ0°¸ZÈÓLˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJà—^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ø"×5Ñ5×<Ñ<¸ZÈÈRÓPÐÜ—Y‘Y 
Ð,?ÀÐLÐRSÔTˆ
á#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^‰Jà˜Ÿ™¨Ñ+Ò+¨u¸¿¹ÈÑ8JÒ/JÜ Ø(¨¨°°%°ð 9ØŸ™¨Ñ+Ð,¨A¨d¯o©o¸aÑ.@Ð-AÀðEóð ð $ d×&>Ñ&>Ñ>ˆJà—\‘\ *Ó-ˆ
ØÐr+   )F)NF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úboolr   r    ÚTensorÚintrH   Ú
BoolTensorr[   Ú__classcell__©r)   s   @r*   r   r   '   sŸ   ø„ ññV˜zð V¸4ð VÈDõ Vð&M°5·<±<ð &MÈð &MÐUXð &MÐ]b×]iÑ]ió &MðV 48Ø).ñ	 à—l‘lð ð ×)Ñ)¨DÑ0ð ð #'ð	 ð
 
�‰÷ r+   r   c                   óx   ‡ — e Zd ZddgZdej
                  ej                  z  ej                  z  ddfˆ fd„Zˆ xZ	S )ÚDeiTPreTrainedModelr   Ú	DeiTLayerÚmoduler   Nc                 óX  •— t         ‰| �  |«       t        |t        «      rŠt	        j
                  |j                  «       t	        j
                  |j                  «       t	        j
                  |j                  «       |j                  � t	        j
                  |j                  «       yyy)zInitialize the weightsN)
r   Ú_init_weightsÚ
isinstancer   ÚinitÚzeros_r#   r&   r'   rM   )r(   ri   r)   s     €r*   rk   z!DeiTPreTrainedModel._init_weightsˆ   sx   ø€ ä‰Ñ˜fÔ%Ü�fœnÔ-Ü�K‰K˜×(Ñ(Ô)Ü�K‰K˜×2Ñ2Ô3Ü�K‰K˜×1Ñ1Ô2Ø× Ñ Ð,Ü—‘˜F×-Ñ-Õ.ð -ð	 .r+   )
r\   r]   r^   Ú_no_split_modulesr   ÚLinearÚConv2dÚ	LayerNormrk   rd   re   s   @r*   rg   rg   …   s<   ø„ Ø)¨;Ð7Ðð/ B§I¡I°·	±	Ñ$9¸B¿L¹LÑ$Hð /ÈT÷ /ñ /r+   rg   c                   ó   — e Zd Zy)Ú	DeiTModelN©r\   r]   r^   © r+   r*   rt   rt   “   ó   „ Ør+   rt   c                   óž   — e Zd Zee	 	 	 	 d	dej                  dz  dej                  dz  dedej                  dz  de	e
   defd„«       «       Zy)
ÚDeiTForMaskedImageModelingNrI   rJ   rH   Úattention_maskÚkwargsr   c                 óh  —  | j                   |f|||dœ|¤Ž}|j                  }|dd…dd…f   }|j                  \  }}	}
t        |	dz  «      x}}|j	                  ddd«      j                  ||
||«      }| j                  |«      }d}|��| j                  j                  | j                  j                  z  }|j                  d||«      }|j                  | j                  j                  d«      j                  | j                  j                  d«      j                  d«      j                  «       }t        j                  j                  ||d¬	«      }||z  j!                  «       |j!                  «       d
z   z  | j                  j"                  z  }t%        |||j&                  |j(                  ¬«      S )a;  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```)rJ   rH   rz   Nr   r1   r   r   r0   Únone)Ú	reductiongñhãˆµøä>)ÚlossÚreconstructionÚhidden_statesÚ
attentions)ÚdeitÚlast_hidden_stater8   rb   r=   r<   Údecoderr   rQ   r;   Úrepeat_interleaverO   Ú
contiguousr   r>   Úl1_lossÚsumÚnum_channelsr   r�   r‚   )r(   rI   rJ   rH   rz   r{   ÚoutputsÚsequence_outputrU   Úsequence_lengthrŠ   r-   r.   Úreconstructed_pixel_valuesÚmasked_im_lossr3   rX   Úreconstruction_losss                     r*   r[   z"DeiTForMaskedImageModeling.forward˜   s«  € ðL /8¨d¯i©iØð/
à+Ø%=Ø)ñ	/
ð
 ñ/
ˆð "×3Ñ3ˆð *ª!¨Q©R¨%Ñ0ˆØ4C×4IÑ4IÑ1ˆ
�O \Ü˜_¨cÑ1Ó2Ð2ˆ�Ø)×1Ñ1°!°Q¸Ó:×BÑBÀ:È|Ð]cÐejÓkˆð &*§\¡\°/Ó%BÐ"àˆØÑ&Ø—;‘;×)Ñ)¨T¯[©[×-CÑ-CÑCˆDØ-×5Ñ5°b¸$ÀÓEˆOà×1Ñ1°$·+±+×2HÑ2HÈ!ÓLß"Ñ" 4§;¡;×#9Ñ#9¸1Ó=ß‘˜1“ß‘“ð	 ô #%§-¡-×"7Ñ"7¸ÐF`ÐlrÐ"7Ó"sÐØ1°DÑ8×=Ñ=Ó?À4Ç8Á8Ã:ÐPTÑCTÑUÐX\×XcÑXc×XpÑXpÑpˆNä(ØØ5Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   )NNFN)r\   r]   r^   r   r   r    ra   rc   r`   r	   r   r   r[   rv   r+   r*   ry   ry   —   s�   „ ØØð -1Ø37Ø).Ø.2ñJ
à—l‘l TÑ)ðJ
ð ×)Ñ)¨DÑ0ðJ
ð #'ð	J
ð
 Ÿ™ tÑ+ðJ
ð Ð+Ñ,ðJ
ð 
#òJ
ó ó ñJ
r+   ry   c                   ó   — e Zd Zy)ÚDeiTForImageClassificationNru   rv   r+   r*   r’   r’   ç   rw   r+   r’   zC
    Output type of [`DeiTForImageClassificationWithTeacher`].
    )Úcustom_introc                   óæ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   y)Ú+DeiTForImageClassificationWithTeacherOutputaj  
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the cls_logits and distillation logits.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    NÚlogitsÚ
cls_logitsÚdistillation_logitsr�   r‚   )r\   r]   r^   r_   r–   r    ÚFloatTensorÚ__annotations__r—   r˜   r�   Útupler‚   rv   r+   r*   r•   r•   ë   s}   … ñ	ð (,€FˆE×Ñ Ñ$Ó+Ø+/€J�×!Ñ! DÑ(Ó/Ø48Ð˜×*Ñ*¨TÑ1Ó8Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r+   r•   aˆ  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                   óš   ‡ — e Zd Zdeddfˆ fd„Zee	 	 	 d
dej                  dz  de	dej                  dz  de
e   def
d	„«       «       Zˆ xZS )Ú%DeiTForImageClassificationWithTeacherr   r   Nc                 óÒ  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j                  dkD  r*t        j                  |j                  |j                  «      nt        j                  «       | _	        |j                  dkD  r*t        j                  |j                  |j                  «      nt        j                  «       | _
        | j                  «        y )NF)Úadd_pooling_layerr   )r   r   Ú
num_labelsrt   rƒ   r   rp   r"   ÚIdentityÚcls_classifierÚdistillation_classifierÚ	post_init)r(   r   r)   s     €r*   r   z.DeiTForImageClassificationWithTeacher.__init__  s·   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô>ˆŒ	ð AG×@QÑ@QÐTUÒ@UŒB�I‰I�f×(Ñ(¨&×*;Ñ*;Ô<Ô[]×[fÑ[fÓ[hð 	Ôð AG×@QÑ@QÐTUÒ@UŒB�I‰I�f×(Ñ(¨&×*;Ñ*;Ô<Ô[]×[fÑ[fÓ[hð 	Ô$ð
 	�‰Õr+   rI   rH   rz   r{   c                 ó
  —  | j                   |f||dœ|¤Ž}|j                  }| j                  |d d …dd d …f   «      }| j                  |d d …dd d …f   «      }||z   dz  }	t	        |	|||j
                  |j                  ¬«      S )N)rH   rz   r   r   r   )r–   r—   r˜   r�   r‚   )rƒ   r„   r¢   r£   r•   r�   r‚   )
r(   rI   rH   rz   r{   r‹   rŒ   r—   r˜   r–   s
             r*   r[   z-DeiTForImageClassificationWithTeacher.forward!  s«   € ð /8¨d¯i©iØð/
à%=Ø)ñ/
ð ñ	/
ˆð "×3Ñ3ˆà×(Ñ(¨º¸Aºq¸Ñ)AÓBˆ
Ø"×:Ñ:¸?Ê1ÈaÒQRÈ7Ñ;SÓTÐð Ð2Ñ2°aÑ7ˆä:ØØ!Ø 3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r+   )NFN)r\   r]   r^   r   r   r   r   r    ra   r`   r	   r   r•   r[   rd   re   s   @r*   r�   r�     s…   ø„ ð˜zð ¨dõ ð" Øð -1Ø).Ø.2ñ	
à—l‘l TÑ)ð
ð #'ð
ð Ÿ™ tÑ+ð	
ð
 Ð+Ñ,ð
ð 
5ò
ó ó ô
r+   r�   )r’   r�   ry   rt   rg   )$r_   Údataclassesr   r    r   Ú r   rm   Úmodeling_outputsr   r   Úprocessing_utilsr	   Úutilsr
   r   r   r   Úutils.genericr   Úvit.modeling_vitr   r   r   r   r   Úconfiguration_deitr   r   rg   rt   ry   r’   r•   r�   Ú__all__rv   r+   r*   ú<module>r¯      sÚ   ðñ >å !ã Ý å &÷õ 'ß OÓ OÝ -÷õ õ +ô[�]ô [ô|/Ð,ô /ô	�ô 	ôM
Ð!:ô M
ô`	Ð!:ô 	ñ ðôð
 ô7°+ó 7ó óð7ñ& ðô
ô0
Ð,?ó 0
ó
ð0
òf�r+   