
    ^j 4                     x   d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZmZmZmZ ddlmZ  G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z  ed      e G d de                    Z! ed       G d de             Z"g d Z#y)!z7PyTorch DeiT (Data-efficient Image Transformers) model.    )	dataclassN)nn   )initialization)BaseModelOutputWithPoolingMaskedImageModelingOutput)Unpack)ModelOutputTransformersKwargsauto_docstring	torch_int)can_return_tuple   )ViTEmbeddingsViTForImageClassificationViTForMaskedImageModelingViTModelViTPreTrainedModel   )
DeiTConfigc            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  dej                  dz  dedej                  fdZ xZS )DeiTEmbeddingsa  
    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.

    Differences from ViTEmbeddings:
    - Adds a distillation token (for distillation pre-training).
    - Position embeddings include +2 slots (CLS + distillation) instead of +1.
    - interpolate_pos_encoding handles 2 special tokens instead of 1.
    - forward concatenates distillation token and handles position encoding for both.
    configuse_mask_tokenreturnNc                    t         |   ||       t        j                  t	        j
                  dd|j                              | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                              | _
        t        j                  t	        j
                  dd|j                              | _        y )N)r   r   r   )super__init__r   	Parametertorchzeroshidden_size	cls_tokenpatch_embeddingsnum_patchesposition_embeddingsdistillation_token)selfr   r   r%   	__class__s       p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/deit/modular_deit.pyr   zDeiTEmbeddings.__init__2   s    ?ekk!Q8J8J&KL++77#%<<A{QPVPbPb0c#d "$,,u{{1aASAS/T"U    
embeddingsheightwidthc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and 2 class embeddings.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   N      ?r   r   bicubicF)sizemodealign_cornersdim)shaper&   r    jit
is_tracing
patch_sizer   reshapepermuter   
functionalinterpolateviewcat)r(   r,   r-   r.   r%   num_positionsclass_and_dist_pos_embedpatch_pos_embedr7   
new_height	new_widthsqrt_num_positionss               r*   interpolate_pos_encodingz'DeiTEmbeddings.interpolate_pos_encoding:   sb    !&&q)A-0066q9A= yy##%+*F6UZ?+++#'#;#;ArrE#B 221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy2OD!LLr+   pixel_valuesbool_masked_posrH   c                    |j                   \  }}}}| j                  |      }|j                         \  }}	}|K| j                  j	                  ||	d      }
|j                  d      j                  |
      }|d|z
  z  |
|z  z   }| j                  j	                  |dd      }| j                  j	                  |dd      }t        j                  |||fd      }|r|| j                  |||      z   }ne|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d		      || j                  z   }| j                  |      }|S )
Nr0   g      ?r   r6   r   zInput image size (*z) doesn't match model (z).)r8   r$   r3   
mask_tokenexpand	unsqueezetype_asr#   r'   r    rA   rH   
image_size
ValueErrorr&   dropout)r(   rI   rJ   rH   _r-   r.   r,   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokenss                 r*   forwardzDeiTEmbeddings.forwardb   s    +001fe**<8
$.OO$5!
J&//00ZLK",,R088ED#sTz2[45GGJ^^**:r2>
"55<<ZRPYY
,?LRST
##d&C&CJPVX]&^^J++u8J/J (% 9+,Adooa.@-AE  $d&>&>>J\\*-
r+   )F)NF)__name__
__module____qualname____doc__r   boolr   r    TensorintrH   
BoolTensorr[   __classcell__r)   s   @r*   r   r   '   s    Vz V4 VD V&M5<< &M &MUX &M]b]i]i &MV 48).	 ll  ))D0  #'	 
 
 r+   r   c                   x     e Zd ZddgZdej
                  ej                  z  ej                  z  ddf fdZ xZ	S )DeiTPreTrainedModelr   	DeiTLayermoduler   Nc                 X   t         |   |       t        |t              rt	        j
                  |j                         t	        j
                  |j                         t	        j
                  |j                         |j                   t	        j
                  |j                         yyy)zInitialize the weightsN)
r   _init_weights
isinstancer   initzeros_r#   r&   r'   rM   )r(   ri   r)   s     r*   rk   z!DeiTPreTrainedModel._init_weights   sx    f%fn-KK(()KK223KK112  ,F--. -	 .r+   )
r\   r]   r^   _no_split_modulesr   LinearConv2d	LayerNormrk   rd   re   s   @r*   rg   rg      s<    );7/BII		$9BLL$H /T / /r+   rg   c                       e Zd Zy)	DeiTModelNr\   r]   r^    r+   r*   rt   rt          r+   rt   c                       e Zd Zee	 	 	 	 d	dej                  dz  dej                  dz  dedej                  dz  de	e
   defd              Zy)
DeiTForMaskedImageModelingNrI   rJ   rH   attention_maskkwargsr   c                 h    | j                   |f|||d|}|j                  }|ddddf   }|j                  \  }}	}
t        |	dz        x}}|j	                  ddd      j                  ||
||      }| j                  |      }d}|| j                  j                  | j                  j                  z  }|j                  d||      }|j                  | j                  j                  d      j                  | j                  j                  d      j                  d      j                         }t        j                  j                  ||d	      }||z  j!                         |j!                         d
z   z  | j                  j"                  z  }t%        |||j&                  |j(                        S )a;  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```)rJ   rH   rz   Nr   r1   r   r   r0   none)	reductiongh㈵>)lossreconstructionhidden_states
attentions)deitlast_hidden_stater8   rb   r=   r<   decoderr   rQ   r;   repeat_interleaverO   
contiguousr   r>   l1_losssumnum_channelsr   r   r   )r(   rI   rJ   rH   rz   r{   outputssequence_outputrU   sequence_lengthr   r-   r.   reconstructed_pixel_valuesmasked_im_lossr3   rX   reconstruction_losss                     r*   r[   z"DeiTForMaskedImageModeling.forward   s   L /8dii/
+%=)	/

 /
 "33 *!QR%04C4I4I1
O\_c122)11!Q:BB:|]cejk &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN(5!//))	
 	
r+   )NNFN)r\   r]   r^   r   r   r    ra   rc   r`   r	   r   r   r[   rv   r+   r*   ry   ry      s     -137)..2J
llT)J
 ))D0J
 #'	J

 t+J
 +,J
 
#J
  J
r+   ry   c                       e Zd Zy)DeiTForImageClassificationNru   rv   r+   r*   r   r      rw   r+   r   zC
    Output type of [`DeiTForImageClassificationWithTeacher`].
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   y)+DeiTForImageClassificationWithTeacherOutputaj  
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores as the average of the cls_logits and distillation logits.
    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
        class token).
    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
        distillation token).
    Nlogits
cls_logitsdistillation_logitsr   r   )r\   r]   r^   r_   r   r    FloatTensor__annotations__r   r   r   tupler   rv   r+   r*   r   r      s}    	 (,FE$++/J!!D(/48**T1859M5**+d2926Je''(4/6r+   r   a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                        e Zd Zdeddf fdZee	 	 	 d
dej                  dz  de	dej                  dz  de
e   def
d	              Z xZS )%DeiTForImageClassificationWithTeacherr   r   Nc                    t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _
        | j                          y )NF)add_pooling_layerr   )r   r   
num_labelsrt   r   r   rp   r"   Identitycls_classifierdistillation_classifier	post_init)r(   r   r)   s     r*   r   z.DeiTForImageClassificationWithTeacher.__init__  s      ++f>	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	$
 	r+   rI   rH   rz   r{   c                 
    | j                   |f||d|}|j                  }| j                  |d d dd d f         }| j                  |d d dd d f         }||z   dz  }	t	        |	|||j
                  |j                        S )N)rH   rz   r   r   r   )r   r   r   r   r   )r   r   r   r   r   r   r   )
r(   rI   rH   rz   r{   r   r   r   r   r   s
             r*   r[   z-DeiTForImageClassificationWithTeacher.forward!  s     /8dii/
%=)/
 	/
 "33((Aq)AB
"::?1aQR7;ST 22a7:! 3!//))
 	
r+   )NFN)r\   r]   r^   r   r   r   r   r    ra   r`   r	   r   r   r[   rd   re   s   @r*   r   r     s    z d "  -1)..2	
llT)
 #'
 t+	

 +,
 
5
  
r+   r   )r   r   ry   rt   rg   )$r_   dataclassesr   r    r    r   rm   modeling_outputsr   r   processing_utilsr	   utilsr
   r   r   r   utils.genericr   vit.modeling_vitr   r   r   r   r   configuration_deitr   r   rg   rt   ry   r   r   r   __all__rv   r+   r*   <module>r      s    > !   & ' O O -  +[] [|/, /	 	M
!: M
`	!: 	 
 7+ 7 7& 
0
,? 0

0
fr+   