Ë
    ÛÓ^jKº  ã                   ó*  — d dl mZmZ d dlmZ d dlZd dlmZmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+  e#d¬«      e G d„ de«      «       «       Z, G d„ dejZ                  «      Z. G d„ dejZ                  «      Z/ G d„ dejZ                  «      Z0	 	 dOdejZ                  dej                  d ej                  d!ej                  d"ej                  dz  d#e1dz  d$e1d%ee"   fd&„Z2 G d'„ d(ejZ                  «      Z3 G d)„ d*ejZ                  «      Z4 G d+„ d,ejZ                  «      Z5 G d-„ d.e«      Z6e# G d/„ d0e«      «       Z7e# G d1„ d2e7«      «       Z8 G d3„ d4ejZ                  «      Z9 e#d5¬«       G d6„ d7e7«      «       Z: e#d8¬«       G d9„ d:e7«      «       Z; G d;„ d<ejZ                  «      Z< G d=„ d>ejZ                  «      Z= G d?„ d@ejZ                  «      Z> G dA„ dBejZ                  «      Z? G dC„ dDejZ                  «      Z@ G dE„ dFejZ                  «      ZA G dG„ dHejZ                  «      ZBe# G dI„ dJe7«      «       ZC e#dK¬«       G dL„ dMee7«      «       ZDg dN¢ZEy)Pé    )ÚCallableÚIterable)Ú	dataclassN)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBackboneOutputÚBaseModelOutputWithPoolingÚImageClassifierOutputÚMaskedLMOutputÚSemanticSegmenterOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Ú#compile_compatible_method_lru_cache)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
BeitConfigz-
    Class for outputs of [`BeitModel`].
    )Úcustom_introc                   ó   — e Zd ZdZy)ÚBeitModelOutputWithPoolingaF  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
        will be returned.
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/beit/modeling_beit.pyr"   r"   0   s   „ òr(   r"   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚBeitPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    Úconfigc                 ó„  •— t         ‰| �  «        |j                  }|j                  }t	        |t
        «      r|n||f}t	        |t
        «      r|n||f}|d   |d   z  |d   |d   z  z  | _        || _        || _        |j                  | _        t        j                  |j                  |j                  ||¬«      | _        y )Nr   r   ©Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚ
isinstancer   Únum_patchesÚnum_channelsr   ÚConv2dÚhidden_sizeÚ
projection)Úselfr,   r3   r4   Ú	__class__s       €r)   r2   zBeitPatchEmbeddings.__init__F   sº   ø€ Ü‰ÑÔØ×&Ñ&ˆ
Ø×&Ñ&ˆ
Ü#-¨j¼(Ô#C‘ZÈ*ÐV`ÐIaˆ
Ü#-¨j¼(Ô#C‘ZÈ*ÐV`ÐIaˆ
à& q™M¨Z¸©]Ñ:¸zÈ!¹}ÐPZÐ[\ÑP]Ñ?]Ñ^ˆÔØ$ˆŒØ$ˆŒØ"×/Ñ/ˆÔÜŸ)™) F×$7Ñ$7¸×9KÑ9KÐYcÐlvÔwˆ�r(   Úpixel_valuesÚreturnc                 óÖ   — |j                   d   }|| j                  k7  rt        d| j                  › d|› d�«      ‚| j                  |«      j	                  d«      j                  dd«      S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.é   )Úshaper7   Ú
ValueErrorr:   ÚflattenÚ	transpose)r;   r=   r7   s      r)   ÚforwardzBeitPatchEmbeddings.forwardS   su   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜðØ!×.Ñ.Ð/¨y¸¸ÀaðIóð ð �‰˜|Ó,×4Ñ4°QÓ7×AÑAÀ!ÀQÓGÐGr(   )
r#   r$   r%   r&   r   r2   Útorchr   rF   Ú__classcell__©r<   s   @r)   r+   r+   ?   s4   ø„ ñðx˜zõ xðH E§L¡Lð H°U·\±\÷ Hr(   r+   c                   óÆ   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dededej                  fd	„Z		 dd
ej                  dej                  dz  dej                  fd„Zˆ xZS )ÚBeitEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r,   r>   Nc                 ól  •— t         ‰| �  «        t        j                  t	        j
                  dd|j                  «      «      | _        |j                  r4t        j                  t	        j
                  dd|j                  «      «      nd | _	        t        |«      | _        |j                  | _        | j                  j                  }|j                  r7t        j                  t	        j
                  d|dz   |j                  «      «      nd | _        t        j                   |j"                  «      | _        y )Nr   )r1   r2   r   Ú	ParameterrG   Úzerosr9   Ú	cls_tokenÚuse_mask_tokenÚ
mask_tokenr+   Úpatch_embeddingsr4   r6   Ú use_absolute_position_embeddingsÚposition_embeddingsÚDropoutÚhidden_dropout_probÚdropout)r;   r,   r6   r<   s      €r)   r2   zBeitEmbeddings.__init__b   sÝ   ø€ Ü‰ÑÔäŸ™¤e§k¡k°!°Q¸×8JÑ8JÓ&KÓLˆŒØQW×QfÒQfœ"Ÿ,™,¤u§{¡{°1°a¸×9KÑ9KÓ'LÔMÐlpˆŒÜ 3°FÓ ;ˆÔØ ×+Ñ+ˆŒØ×+Ñ+×7Ñ7ˆð ×6Ò6ô �L‰LœŸ™ Q¨°a©¸×9KÑ9KÓLÔMàð 	Ô ô
 —z‘z &×"<Ñ"<Ó=ˆ�r(   Ú
embeddingsÚheightÚwidthc                 ó¦  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }t        j                  j                  ||	|
fdd	¬
«      }|j                  dddd«      j                  dd|«      }t        j                  ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Néÿÿÿÿg      à?r   r   rA   ÚbicubicF©ÚsizeÚmodeÚalign_corners©Údim)rB   rT   rG   ÚjitÚ
is_tracingr4   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r;   rX   rY   rZ   r6   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrc   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r)   Úinterpolate_pos_encodingz'BeitEmbeddings.interpolate_pos_encodingq   s`  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×2Ñ2²1°b°q°b°5Ñ9ˆØ×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr(   r=   Úbool_masked_posc                 óÞ  — |j                   \  }}}}| j                  |«      }|j                  «       \  }}}|�K| j                  j	                  ||d«      }	|j                  d«      j                  |	«      }
|d|
z
  z  |	|
z  z   }| j                  j	                  |dd«      }t        j                  ||fd¬«      }| j                  �|| j                  |||«      z   }| j                  |«      }|S ©Nr\   r   rb   )rB   rR   r_   rQ   ÚexpandÚ	unsqueezeÚtype_asrO   rG   rk   rT   rr   rW   )r;   r=   rs   Ú_rY   rZ   rX   Ú
batch_sizeÚseq_lenÚmask_tokensÚmaskÚ
cls_tokenss               r)   rF   zBeitEmbeddings.forward™   sô   € ð
 +×0Ñ0Ñˆˆ1ˆf�eØ×*Ñ*¨<Ó8ˆ
Ø!+§¡Ó!2Ñˆ
�G˜QàÐ&ØŸ/™/×0Ñ0°¸WÀbÓIˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# q¨4¡xÑ0°;ÀÑ3EÑEˆJà—^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ü—Y‘Y 
¨JÐ7¸QÔ?ˆ
à×#Ñ#Ð/Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJà—\‘\ *Ó-ˆ
àÐr(   ©N)r#   r$   r%   r&   r   r2   rG   r   Úintrr   Ú
BoolTensorrF   rH   rI   s   @r)   rK   rK   ]   s‡   ø„ ñð>˜zð >¨dõ >ð&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]ió &DðV 48ñà—l‘lðð ×)Ñ)¨DÑ0ðð 
�‰÷	r(   rK   c                   óž   ‡ — e Zd Zdeddfˆ fd„Ze ed¬«      deeef   de	j                  fd„«       «       Zdd	ede	j                  fd
„Zˆ xZS )ÚBeitRelativePositionBiasr,   r>   Nc                 óª  •— t         ‰| �  «        |j                  }t        |t        t
        f«      s||f}|d   |j                  z  |d   |j                  z  f| _        d| j                  d   z  dz
  d| j                  d   z  dz
  z  dz   | _        t        j                  t        j                  | j                  |j                  «      «      | _        y ©Nr   r   rA   r   )r1   r2   r3   r5   ÚtupleÚlistr4   Úwindow_sizeÚnum_relative_distancer   rM   rG   rN   Únum_attention_headsÚrelative_position_bias_table)r;   r,   r3   r<   s      €r)   r2   z!BeitRelativePositionBias.__init__´   sÇ   ø€ Ü‰ÑÔØ×&Ñ&ˆ
Ü˜*¤u¬d mÔ4Ø$ jÐ1ˆJØ& q™M¨V×->Ñ->Ñ>À
È1ÁÐQW×QbÑQbÑ@bÐcˆÔØ&'¨$×*:Ñ*:¸1Ñ*=Ñ&=ÀÑ&AÀaÈ$×JZÑJZÐ[\ÑJ]ÑF]Ð`aÑFaÑ%bÐefÑ%fˆÔ"Ü,.¯L©LÜ�K‰K˜×2Ñ2°F×4NÑ4NÓOó-
ˆÕ)r(   é
   )Úmaxsizerˆ   c                 óþ  — d| d   z  dz
  d| d   z  dz
  z  dz   }| d   | d   z  }t        j                  t        j                  t        j                  t        j                  | d   «      t        j                  | d   «      d¬«      «      d¬«      }|dd…dd…df   |dd…ddd…f   z
  j                  ddd«      j                  «       }|dd…dd…dfxx   | d   dz
  z  cc<   |dd…dd…dfxx   | d   dz
  z  cc<   |dd…dd…dfxx   d| d   z  dz
  z  cc<   t        j                  |dz   fdz  |j                  ¬	«      }|j                  d
«      |dd…dd…f<   |dz
  |ddd…f<   |dz
  |dd…df<   |dz
  |d<   |S )z¿
        This method creates the relative position index, modified to support arbitrary window sizes,
        as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
        rA   r   r   r   Úij)Úindexing)Ú	start_dimN)r_   Údtyper\   )r   r   )
rG   rD   ÚstackÚmeshgridÚarangerg   Ú
contiguousrN   r’   Úsum)rˆ   r‰   Úwindow_areaÚcoords_flattenÚrelative_coordsÚrelative_position_indexs         r)   Ú generate_relative_position_indexz9BeitRelativePositionBias.generate_relative_position_indexÀ   s²  € ð "# [°¡^Ñ!3°aÑ!7¸AÀÈAÁÑ<NÐQRÑ<RÑ SÐVWÑ WÐØ! !‘n {°1¡~Ñ5ˆô Ÿ™Ü�K‰KœŸ™¤u§|¡|°KÀ±NÓ'CÄUÇ\Á\ÐR]Ð^_ÑR`ÓEaÐlpÔqÓrØô
ˆð *ª!ªQ°¨*Ñ5¸ÂqÈ$ÒPQÀzÑ8RÑR×[Ñ[Ð\]Ð_`ÐbcÓd×oÑoÓqˆàšš1˜a˜Ó  K°¡N°QÑ$6Ñ6Ó Øšš1˜a˜Ó  K°¡N°QÑ$6Ñ6Ó Øšš1˜a˜Ó  A¨°A©Ñ$6¸Ñ$:Ñ:Ó ä"'§+¡+°KÀ!±OÐ3EÈÑ3IÐQ`×QfÑQfÔ"gÐØ*9×*=Ñ*=¸bÓ*AÐ ¡ A¡B Ñ'Ø)>ÀÑ)BÐ  1¡2 Ñ&Ø)>ÀÑ)BÐ ¡ A Ñ&Ø(=ÀÑ(AÐ Ñ%Ø&Ð&r(   rr   c                 ó   — d| j                   d   z  dz
  }d| j                   d   z  dz
  }d|d   z  dz
  }d|d   z  dz
  }| j                  }| j                  }	||z  dz   }
|d|	dz
   }|j                  d||d«      j	                  dddd«      }t
        j                  j                  |t        |«      t        |«      fd¬«      }|j	                  dddd«      j                  |
dz
  d«      }t        j                  |||	dz
  d g«      }| j                  |«      }||j                  d«         }|j                  |d   |d   z  dz   |d   |d   z  dz   d«      }|j	                  ddd«      j                  «       }|rCt
        j                  j                  |j                  d«      ||fdd	¬
«      j                  d«      }|j                  d«      S )zu
        Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
        rA   r   r   r   Nr\   Úbilinear)r_   r`   Fr^   )rˆ   r‹   r‰   rf   rg   r   rh   ri   r   rG   rk   rœ   rj   r–   rw   Úsqueeze)r;   rˆ   rr   Údim_sizeÚ
old_heightÚ	old_widthro   rp   Ú old_relative_position_bias_tableÚold_num_relative_distanceÚnew_num_relative_distanceÚold_sub_tableÚnew_sub_tableÚ new_relative_position_bias_tabler›   Úrelative_position_biass                   r)   rF   z BeitRelativePositionBias.forwardÜ   s-  € ð ˜×)Ñ)¨!Ñ,Ñ,¨qÑ0ˆ
Ø˜×(Ñ(¨Ñ+Ñ+¨aÑ/ˆ	à˜ Q™Ñ'¨!Ñ+ˆ
Ø˜ A™Ñ&¨Ñ*ˆ	à+/×+LÑ+LÐ(à$(×$>Ñ$>Ð!Ø$.°Ñ$:¸QÑ$>Ð!à8Ð9XÐ;TÐWXÑ;XÐYˆà%×-Ñ-¨a°¸JÈÓK×SÑSÐTUÐWXÐZ[Ð]^Ó_ˆÜŸ™×1Ñ1Ø¤¨:Ó!6¼	À)Ó8LÐ MÐT^ð 2ó 
ˆð &×-Ñ-¨a°°A°qÓ9×AÑAÐB[Ð^_ÑB_ÐacÓdˆä+0¯9©9ØÐ<Ð=VÐYZÑ=ZÐ=\Ð]Ð^ó,
Ð(ð #'×"GÑ"GÈÓ"TÐØ!AÐBY×B^ÑB^Ð_aÓBbÑ!cÐð "8×!<Ñ!<Ø˜‰N˜[¨™^Ñ+¨aÑ/°¸Q±À+ÈaÁ.Ñ1PÐSTÑ1TÐVXó"
Ðð "8×!?Ñ!?ÀÀ1ÀaÓ!H×!SÑ!SÓ!UÐá#Ü%'§]¡]×%>Ñ%>Ø&×0Ñ0°Ó3Ø Ð)ØØ#ð	 &?ó &÷
 ‰g�a‹jð #ð &×/Ñ/°Ó2Ð2r(   )FN)r#   r$   r%   r   r2   Ústaticmethodr   r†   r€   rG   r   rœ   ÚboolrF   rH   rI   s   @r)   rƒ   rƒ   ³   sk   ø„ ð	
˜zð 	
¨dõ 	
ð Ù(°Ô4ð'°e¸CÀ¸H±oð 'È%Ï,É,ò 'ó 5ó ð'ñ4-3¸Tð -3Ð]b×]iÑ]i÷ -3r(   rƒ   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrW   Úkwargsc                 óì  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr\   ç      à¿rA   r   )rc   r’   )ÚpÚtrainingr   )r_   rG   ÚmatmulrE   r   rh   ÚsoftmaxÚfloat32Útor’   rW   r¶   r–   )
r¬   r­   r®   r¯   r°   r±   rW   r²   Úattn_weightsÚattn_outputs
             r)   Úeager_attention_forwardr½     sÓ   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r(   c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee   de	ej                  ej                  f   fd„Z
ˆ xZS )
ÚBeitAttentionr,   c                 óà  •— t         ‰| �  «        || _        |j                  | _        t	        |d|j
                  |j                  z  «      | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j
                  |j                  | j                  z  «      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  «      | _        t        j                  |j                  | j                  z  |j
                  «      | _        y )NÚhead_dimr´   F)Úbias)r1   r2   r,   rŠ   Úgetattrr9   rÁ   Úattention_probs_dropout_probÚattention_dropoutr±   Ú	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj©r;   r,   r<   s     €r)   r2   zBeitAttention.__init__)  s  ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ!'×!DÑ!DˆÔØ—}‘} dÑ*ˆŒØˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ó_ˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ó_ˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^Ó_ˆ�r(   NÚhidden_statesr°   r²   r>   c                 óš  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  sdn| j                  | j                  dœ|¤Ž\  }
} |
j                  g |¢d‘­Ž j!                  «       }
| j#                  |
«      }
|
|fS )Nr\   r   rA   ç        )rW   r±   )rB   rÁ   rÈ   rj   rE   rÉ   rÊ   r   Úget_interfacer,   Ú_attn_implementationr½   r¶   rÅ   r±   rf   r–   rË   )r;   rÍ   r°   r²   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer¼   r»   s               r)   rF   zBeitAttention.forward6  sK  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆà˜LÐ(Ð(r(   r   )r#   r$   r%   r   r2   rG   r   r   r   r†   rF   rH   rI   s   @r)   r¿   r¿   (  sf   ø„ ð`˜zõ `ð  /3ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð Ð+Ñ,ð	)ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷)r(   r¿   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚBeitMLPr,   c                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y r   )r1   r2   r,   r
   Ú
hidden_actÚactivation_fnr   rÇ   r9   Úintermediate_sizeÚfc1Úfc2rÌ   s     €r)   r2   zBeitMLP.__init__Y  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r(   rÍ   r>   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r   )rÞ   rÜ   rß   ©r;   rÍ   s     r)   rF   zBeitMLP.forward`  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆàÐr(   ©	r#   r$   r%   r   r2   rG   r   rF   rH   rI   s   @r)   rÙ   rÙ   X  s,   ø„ ðK˜zõ Kð U§\¡\ð °e·l±l÷ r(   rÙ   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚBeitDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr>   Nc                 ó0   •— t         ‰| �  «        || _        y r   )r1   r2   rå   )r;   rå   r<   s     €r)   r2   zBeitDropPath.__init__o  s   ø€ Ü‰ÑÔØ"ˆ�r(   rÍ   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )NrÏ   r   r   )r   )r’   Údevice)
rå   r¶   rB   ÚndimrG   Úrandr’   rè   ÚfloorÚdiv)r;   rÍ   Ú	keep_probrB   Úrandom_tensors        r)   rF   zBeitDropPath.forwards  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r(   c                 ó    — d| j                   › �S )Nzp=)rå   ©r;   s    r)   Ú
extra_reprzBeitDropPath.extra_repr|  s   € Ø�D—N‘NÐ#Ð$Ð$r(   ©rÏ   )r#   r$   r%   r&   Úfloatr2   rG   r   rF   Ústrrñ   rH   rI   s   @r)   rä   rä   h  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r(   rä   c                   ó®   ‡ — e Zd ZdZddedefˆ fd„Z	 	 	 ddej                  dej                  dz  de	d	e
eef   dz  d
ee   dej                  fd„Zˆ xZS )Ú	BeitLayerz?This corresponds to the Block class in the timm implementation.r,   Údrop_path_ratec                 óJ  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  ¬«      | _        t	        j
                  |j                  |j                  ¬«      | _	        t        |«      | _        t	        j                  |j                  «      | _        |j                  | _        |dkD  rt!        |«      nt	        j"                  «       | _        |j&                  }|dkD  r7t	        j(                  |t+        j,                  |j                  «      z  d¬«      nd| _        |dkD  r7t	        j(                  |t+        j,                  |j                  «      z  d¬«      nd| _        |j2                  rt5        |«      | _        y d | _        y )N©ÚepsrÏ   r   T)Úrequires_gradg      ð?)r1   r2   r¿   Ú	attentionr   Ú	LayerNormr9   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterrÙ   ÚmlprU   rV   rW   r4   rä   ÚIdentityÚ	drop_pathÚlayer_scale_init_valuerM   rG   ÚonesÚlambda_1Úlambda_2Úuse_relative_position_biasrƒ   r©   )r;   r,   r÷   Úinit_valuesr<   s       €r)   r2   zBeitLayer.__init__ƒ  s@  ø€ Ü‰ÑÔÜ& vÓ.ˆŒÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜ˜6“?ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØ9GÈ#Ò9Mœ nÔ5ÔSU×S^ÑS^ÓS`ˆŒà×3Ñ3ˆà^iÐlmÒ^mŒB�L‰L˜¤u§z¡z°&×2DÑ2DÓ'EÑEÐUYÕZÐsvð 	Œð _jÐlmÒ^mŒB�L‰L˜¤u§z¡z°&×2DÑ2DÓ'EÑEÐUYÕZÐsvð 	Œð KQ×JkÒJkÔ&>¸vÓ&FˆÕ#ÐquˆÕ#r(   NrÍ   r°   rr   Ú
resolutionr²   r>   c                 ó&  — | j                   �M|\  }}|| j                  z  || j                  z  f}| j                  |||j                  d   ¬«      }	|�|	|z   n|	}|}
| j                  |«      } | j                  |fd|i|¤Ž\  }}| j                  |«      }| j                  |z  }| j                  |«      |
z   }|}
| j                  |«      }| j                  |«      }| j                  |«      }| j                  |z  }| j                  |«      |
z   }|S )Nr   )r    r°   )r©   r4   rB   rÿ   rü   rW   r  r  r   r  r  )r;   rÍ   r°   rr   r
  r²   rY   rZ   rˆ   r©   Úresidualry   s               r)   rF   zBeitLayer.forward–  sE  € ð ×&Ñ&Ð2Ø&‰MˆF�EØ! T§_¡_Ñ4°e¸t¿¹Ñ6NÐOˆKØ%)×%@Ñ%@ØÐ5À×@SÑ@SÐTUÑ@Vð &Aó &Ð"ð <JÐ;UÐ&¨Ò7Ð[qð ð
 !ˆØ×-Ñ-¨mÓ<ˆØ)˜4Ÿ>™>Øñ
à)ð
ð ñ
Ñˆ�qð
 Ÿ™ ]Ó3ˆØŸ™¨Ñ5ˆØŸ™ }Ó5¸Ñ@ˆð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆØŸ™¨Ñ5ˆØŸ™ }Ó5¸Ñ@ˆàÐr(   rò   ©NFN)r#   r$   r%   r&   r   ró   r2   rG   r   r«   r†   r€   r   r   rF   rH   rI   s   @r)   rö   rö   €  sŠ   ø„ ÙIñv˜zð v¸5õ vð, /3Ø).Ø-1ñ&à—|‘|ð&ð Ÿ™ tÑ+ð&ð #'ð	&ð
 ˜#˜s˜(‘O dÑ*ð&ð Ð+Ñ,ð&ð 
�‰÷&r(   rö   c                   ó�   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZdZeedœZd	Zd
gZ ej*                  «       ˆ fd„«       Zˆ xZS )ÚBeitPreTrainedModelr,   Úbeitr=   )ÚimageTrö   F)rÍ   Ú
attentionsrR   z.*relative_position_index.*c                 óÎ  •— t         ‰| �  |«       t        |t        «      rwt	        j
                  |j                  «       |j                  �t	        j
                  |j                  «       |j                  � t	        j
                  |j                  «       yyt        |t        «      r t	        j
                  |j                  «       yt        |t        «      rŽt        |j                  t        j                  «      rit	        j                  |j                  | j                   j"                  «       t	        j                  |j$                  | j                   j"                  «       yyy)zInitialize the weightsN)r1   Ú_init_weightsr5   rK   ÚinitÚzeros_rO   rQ   rT   rƒ   r‹   rö   r  r   rM   Ú	constant_r,   r  r  )r;   r¬   r<   s     €r)   r  z!BeitPreTrainedModel._init_weightsÓ  sê   ø€ ô 	‰Ñ˜fÔ%Ü�fœnÔ-Ü�K‰K˜×(Ñ(Ô)Ø× Ñ Ð,Ü—‘˜F×-Ñ-Ô.Ø×)Ñ)Ð5Ü—‘˜F×6Ñ6Õ7ð 6ä˜Ô 8Ô9Ü�K‰K˜×;Ñ;Õ<Ü˜¤	Ô*Ü˜&Ÿ/™/¬2¯<©<Ô8Ü—‘˜vŸ™°·±×0RÑ0RÔSÜ—‘˜vŸ™°·±×0RÑ0RÕSð 9ð +r(   )r#   r$   r%   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphrö   r¿   Ú_can_record_outputsÚ_input_embed_layerÚ"_keys_to_ignore_on_load_unexpectedrG   Úno_gradr  rH   rI   s   @r)   r  r  ¿  s€   ø… àÓØÐØ$€OØ!ÐØ&*Ð#Ø$˜ÐØ€NØ ÐØÐØ"&ÐØ!Ðà"Ø#ñÐð ,ÐØ*HÐ)IÐ&à€U‡]�]ƒ_óTó ôTr(   r  c                   óÐ   ‡ — e Zd Zddededdfˆ fd„Ze ed¬«      e	 	 	 dde	j                  d	e	j                  dz  d
ede	j                  dz  dee   defd„«       «       «       Zˆ xZS )Ú	BeitModelr,   Úadd_pooling_layerr>   Nc           	      ó‚  •— t         ‰| �  |«       || _        t        |«      | _        |j
                  rt        |«      nd| _        t        |j                  «      D �cg c]+  }|j                  |z  t        |j                  dz
  d«      z  ‘Œ- }}t        j                  |D �cg c]  }t        ||¬«      ‘Œ c}«      | _        |j                   rt        j"                  «       n*t        j$                  |j&                  |j(                  ¬«      | _        |rt-        |«      nd| _        | j1                  «        yc c}w c c}w )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        Nr   )r÷   rù   )r1   r2   r,   rK   rX   Ú!use_shared_relative_position_biasrƒ   Úshared_position_biasÚrangeÚnum_hidden_layersr÷   Úmaxr   Ú
ModuleListrö   ÚlayersÚuse_mean_poolingr  rý   r9   rþ   Ú	layernormÚ
BeitPoolerÚpoolerÚ	post_init)r;   r,   r)  ÚiÚdrop_path_ratesÚrr<   s         €r)   r2   zBeitModel.__init__ç  s  ø€ ô
 	‰Ñ˜Ô ØˆŒä(¨Ó0ˆŒà06×0XÒ0XÔ$ VÔ,Ð^bð 	Ô!ô W\Ð\b×\tÑ\tÓVuö
ØQRˆF×!Ñ! AÑ%¬¨F×,DÑ,DÀqÑ,HÈ!Ó(LÓLð
ˆð 
ô —m‘mÐRaÖ$bÈQ¤Y¨vÀaÖ%HÒ$bÓcˆŒð $×4Ò4ŒB�K‰KŒM¼"¿,¹,Àv×GYÑGYÐ_e×_tÑ_tÔ:uð 	Œñ ->”j Ô(À4ˆŒð 	�‰Õùò
ùò %cs   Á0D7Â"D<F)Útie_last_hidden_statesr=   rs   rr   r°   r²   c                 ó
  — | j                  ||¬«      }|j                  dd }t        | j                  ||¬«      }| j                  �a|\  }}	|| j                  j
                  z  |	| j                  j
                  z  f}
| j	                  |
||j                  d   ¬«      }|�||z   n|}|}| j                  D ]  } ||f|||dœ|¤Ž}Œ | j                  |«      }| j                  �| j                  |«      nd}t        ||¬«      S )	zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        )rs   rA   N)r,   Úinputs_embedsr°   r   )rr   r    )r°   rr   r
  )Úlast_hidden_stateÚpooler_output)
rX   rB   r   r,   r,  r4   r1  r3  r5  r"   )r;   r=   rs   rr   r°   r²   Úembedding_outputr
  rY   rZ   rˆ   Úshared_relative_position_biasrÍ   ÚlayerÚsequence_outputÚpooled_outputs                   r)   rF   zBeitModel.forward   s<  € ð  Ÿ?™?¨<È˜?ÓYÐØ!×'Ñ'¨¨Ð+ˆ
ä2Ø—;‘;Ø*Ø)ô
ˆð ×$Ñ$Ð0Ø&‰MˆF�EØ! T§[¡[×%;Ñ%;Ñ;¸UÀdÇkÁk×F\ÑF\Ñ=\Ð]ˆKØ,0×,EÑ,EØÐ6NÐYi×YoÑYoÐpqÑYrð -Fó -Ð)ð
 "Ð-ð .°Ò>à2ð ð )ˆØ—[‘[ò 	ˆEÙ!Øðà-Ø)AØ%ñ	ð
 ñ‰Mð	ð Ÿ.™.¨Ó7ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆä)¸OÐ[hÔiÐir(   )Tr  )r#   r$   r%   r   r«   r2   r   r   r   rG   r   r�   r   r   r"   rF   rH   rI   s   @r)   r(  r(  å  s±   ø„ ñ˜zð ¸dð Èdõ ð2  Ù¨EÔ2Øð 48Ø).Ø.2ñ-jà—l‘lð-jð ×)Ñ)¨DÑ0ð-jð #'ð	-jð
 Ÿ™ tÑ+ð-jð Ð+Ñ,ð-jð 
$ò-jó ó 3ó  ô-jr(   r(  c                   ó`   ‡ — e Zd Zdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZS )r4  r,   r>   Nc                 óª   •— t         ‰| �  «        |j                  r1t        j                  |j
                  |j                  ¬«      | _        y d | _        y )Nrù   )r1   r2   r2  r   rý   r9   rþ   r3  rÌ   s     €r)   r2   zBeitPooler.__init__4  sA   ø€ Ü‰ÑÔàKQ×KbÒKbŒB�L‰L˜×+Ñ+°×1FÑ1FÔGð 	�Øhlð 	�r(   rÍ   c                 ó„   — | j                   �,| j                  |d d …dd …d d …f   j                  d«      «      S |d d …df   S )Nr   r   )r3  Úmeanrá   s     r)   rF   zBeitPooler.forward:  sD   € àBFÇ.Á.ÐB\ˆt�~‰~˜mªA¨q©r²1¨HÑ5×:Ñ:¸1Ó=Ó>ÐuÐboÒpqÐstÐptÑbuÐur(   râ   rI   s   @r)   r4  r4  3  s4   ø„ ð
˜zð 
¨dõ 
ðv U§\¡\ð v°e·l±l÷ vr(   r4  aÔ  
    Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting
    visual tokens of a Vector-Quantize Variational Autoencoder (VQ-VAE), whereas other vision models like ViT and DeiT
    predict RGB pixel values. As a result, this class is incompatible with [`AutoModelForMaskedImageModeling`], so you
    will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT.
    c                   óæ   ‡ — e Zd Zdeddfˆ fd„Zd„ Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ed
ej                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚBeitForMaskedImageModelingr,   r>   Nc                 óH  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y )NF©r)  rù   )r1   r2   Ú
num_labelsr(  r  r   rý   r9   rþ   r3  rÇ   Ú
vocab_sizeÚlm_headr6  rÌ   s     €r)   r2   z#BeitForMaskedImageModeling.__init__H  su   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô>ˆŒ	ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÓGˆŒð 	�‰Õr(   c                  ó   — y r   r'   rð   s    r)   Úget_output_embeddingsz0BeitForMaskedImageModeling.get_output_embeddingsU  s   € Ør(   r=   rs   Úlabelsrr   r°   r²   c                 ó,  —  | j                   |f|||dœ|¤Ž}|j                  }| j                  |«      }| j                  |dd…dd…f   «      }	d}
|� t	        j
                  «       } ||	|   |«      }
t        |
|	|j                  |j                  ¬«      S )a…  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
        >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, logits = outputs.loss, outputs.logits
        >>> list(logits.shape)
        [1, 196, 8192]
        ```)rs   rr   r°   Nr   ©ÚlossÚlogitsrÍ   r  )	r  r=  r3  rN  r   ÚCrossEntropyLossr   rÍ   r  )r;   r=   rs   rQ  rr   r°   r²   ÚoutputsrB  Úprediction_scoresÚmasked_lm_lossÚloss_fcts               r)   rF   z"BeitForMaskedImageModeling.forwardX  sµ   € ðX �$—)‘)Øð
à+Ø%=Ø)ñ	
ð
 ñ
ˆð "×3Ñ3ˆØŸ.™.¨Ó9ˆØ ŸL™L¨º¸A¹B¸Ñ)?Ó@ÐàˆØÐÜ×*Ñ*Ó,ˆHÙ%Ð&7¸Ñ&HÈ&ÓQˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   )NNNFN)r#   r$   r%   r   r2   rP  r   r   rG   r   r�   r«   r   r   r†   r   rF   rH   rI   s   @r)   rI  rI  ?  sÆ   ø„ ð˜zð ¨dõ òð Øð -1Ø37Ø&*Ø).Ø.2ñ@
à—l‘l TÑ)ð@
ð ×)Ñ)¨DÑ0ð@
ð —‘˜tÑ#ð	@
ð
 #'ð@
ð Ÿ™ tÑ+ð@
ð Ð+Ñ,ð@
ð 
�Ñ	ò@
ó ó ô@
r(   rI  z¶
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                   ó    ‡ — e Zd Zdeddfˆ fd„Zee	 	 	 d
dej                  dz  dej                  dz  de	de
e   deez  f
d	„«       «       Zˆ xZS )ÚBeitForImageClassificationr,   r>   Nc                 ó.  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j                  dkD  r*t        j                  |j                  |j                  «      nt        j                  «       | _	        | j                  «        y )NTrK  r   )r1   r2   rL  r(  r  r   rÇ   r9   r  Ú
classifierr6  rÌ   s     €r)   r2   z#BeitForImageClassification.__init__¤  ss   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô=ˆŒ	ð OU×N_ÑN_ÐbcÒNcœ"Ÿ)™) F×$6Ñ$6¸×8IÑ8IÔJÔik×itÑitÓivˆŒð 	�‰Õr(   r=   rQ  rr   r²   c                 óî   —  | j                   |fd|i|¤Ž}|j                  }| j                  |«      }d}|�| j                  ||| j                  «      }t        |||j                  |j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rr   NrS  )r  r>  r^  Úloss_functionr,   r   rÍ   r  )	r;   r=   rQ  rr   r²   rW  rC  rU  rT  s	            r)   rF   z"BeitForImageClassification.forward°  sŒ   € ð �$—)‘)Øñ
à%=ð
ð ñ
ˆð  ×-Ñ-ˆà—‘ Ó/ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   ©NNF)r#   r$   r%   r   r2   r   r   rG   r   r«   r   r   r†   r   rF   rH   rI   s   @r)   r\  r\  �  s‹   ø„ ð
˜zð 
¨dõ 
ð Øð -1Ø&*Ø).ñ	 
à—l‘l TÑ)ð 
ð —‘˜tÑ#ð 
ð #'ð	 
ð
 Ð+Ñ,ð 
ð 
Ð&Ñ	&ò 
ó ó ô 
r(   r\  c                   óÂ   ‡ — e Zd Z	 	 	 	 	 	 	 ddededeeeef   z  dedeeeef   z  ez  dedeeeef   z  ded	efˆ fd
„Zdej                  dej                  fd„Z
ˆ xZS )ÚBeitConvLayerÚin_channelsÚout_channelsr/   r0   ÚpaddingrÂ   ÚdilationÚgroupsÚ
activationc
           
      óî   •— t         ‰
| �  «        t        j                  ||||||||¬«      | _        t        j
                  |«      | _        |	�t        |	   | _	        y t        j                  «       | _	        y )N)rd  re  r/   r0   rf  rg  rh  rÂ   )
r1   r2   r   r8   ÚconvolutionÚBatchNorm2dÚnormalizationr
   r  ri  )r;   rd  re  r/   r0   rf  rÂ   rg  rh  ri  r<   s             €r)   r2   zBeitConvLayer.__init__Ö  si   ø€ ô 	‰ÑÔÜŸ9™9Ø#Ø%Ø#ØØØØØô	
ˆÔô  Ÿ^™^¨LÓ9ˆÔØ0:Ð0Fœ& Ñ,ˆ�ÌBÏKÉKËMˆ�r(   rÍ   r>   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r   )rk  rm  ri  rá   s     r)   rF   zBeitConvLayer.forwardð  s6   € Ø×(Ñ(¨Ó7ˆØ×*Ñ*¨=Ó9ˆØŸ™¨Ó6ˆØÐr(   )r   r   r   Fr   r   Úrelu)r#   r$   r%   r€   r†   rô   r«   r2   rG   r   rF   rH   rI   s   @r)   rc  rc  Õ  sÈ   ø„ ð
 ./ØØ/0ØØ*+ØØ ñZàðZð ðZð ˜5  c ™?Ñ*ð	Zð
 ðZð �u˜S #˜X‘Ñ&¨Ñ,ðZð ðZð ˜˜c 3˜h™Ñ'ðZð ðZð õZð4 U§\¡\ð °e·l±l÷ r(   rc  c                   óv   ‡ — e Zd Zdedededdfˆ fd„Zdej                  deeef   dej                  fd	„Zˆ xZ	S )
ÚBeitPyramidPoolingBlockÚ
pool_scalerd  Úchannelsr>   Nc                 ó|   •— t         ‰| �  «        t        j                  |«      | _        t        ||d¬«      | _        y )Nr   ©r/   )r1   r2   r   ÚAdaptiveAvgPool2dÚpoolingrc  Úconv)r;   rr  rd  rs  r<   s       €r)   r2   z BeitPyramidPoolingBlock.__init__ø  s0   ø€ Ü‰ÑÔÜ×+Ñ+¨JÓ7ˆŒÜ! +¨xÀQÔGˆ�	r(   Úinputr_   c                 ó�   — | j                  |«      }| j                  |«      }t        j                  j	                  ||dd¬«      }|S )Nrž   Fr^   )rw  rx  r   rh   ri   )r;   ry  r_   Úhidden_states       r)   rF   zBeitPyramidPoolingBlock.forwardý  sB   € Ø—|‘| EÓ*ˆØ—y‘y Ó.ˆÜ—}‘}×0Ñ0°ÀDÈzÐinÐ0ÓoˆØÐr(   )
r#   r$   r%   r€   r2   rG   r   r†   rF   rH   rI   s   @r)   rq  rq  ÷  sS   ø„ ðH 3ð H°Sð HÀCð HÈDõ Hð
˜UŸ\™\ð °°s¸C°x±ð ÀUÇ\Á\÷ r(   rq  c                   ó|   ‡ — e Zd ZdZdeedf   dededdfˆ fd„Zd	ej                  de	ej                     fd
„Z
ˆ xZS )ÚBeitPyramidPoolingModuleak  
    Pyramid Pooling Module (PPM) used in PSPNet.

    Args:
        pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
            Module.
        in_channels (int): Input channels.
        channels (int): Channels after modules, before conv_seg.

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    Úpool_scales.rd  rs  r>   Nc           
      ó¾   •— t         ‰| �  «        || _        || _        || _        t        j                  |D �cg c]  }t        |||¬«      ‘Œ c}«      | _        y c c}w )N)rr  rd  rs  )	r1   r2   r~  rd  rs  r   r0  rq  Úblocks)r;   r~  rd  rs  rr  r<   s        €r)   r2   z!BeitPyramidPoolingModule.__init__  s\   ø€ Ü‰ÑÔØ&ˆÔØ&ˆÔØ ˆŒÜ—m‘mð #.öàô (°:È;ÐaiÖjòó
ˆ�ùòs   ¸ArÍ   c                 óv   — |j                  «       dd  }| j                  D �cg c]  } |||¬«      ‘Œ c}S c c}w )NrA   )r_   )r_   r€  )r;   rÍ   Úoriginal_sizeÚblocks       r)   rF   z BeitPyramidPoolingModule.forward  s6   € Ø%×*Ñ*Ó,¨Q¨RÐ0ˆØFJÇkÁkÖR¸U‘�m¨-Ö8ÒRÐRùÒRs   ¢6)r#   r$   r%   r&   r†   r€   r2   rG   r   r‡   rF   rH   rI   s   @r)   r}  r}    sV   ø„ ñ
ð

 E¨#¨s¨(¡Oð 

À#ð 

ÐQTð 

ÐY]õ 

ðS U§\¡\ð S°d¸5¿<¹<Ñ6H÷ Sr(   r}  c                   ó¨   ‡ — e Zd ZdZdeddfˆ fd„Zdeej                     dej                  fd„Z	deej                     dej                  fd	„Z
ˆ xZS )
ÚBeitUperHeadzú
    Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
    [UPerNet](https://huggingface.co/papers/1807.10221).

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r,   r>   Nc           	      óÖ  •— t         ‰| �  «        |j                  | _        |j                  gdz  | _        |j                  | _        t        j                  | j
                  |j                  d¬«      | _	        t        | j                  | j                  d   | j
                  «      | _        t        | j                  d   t        | j                  «      | j
                  z  z   | j
                  dd¬«      | _        t        j                  «       | _        t        j                  «       | _        | j                  d d D ]o  }| j                   j%                  t        || j
                  d¬«      «       | j"                  j%                  t        | j
                  | j
                  dd¬«      «       Œq t        t        | j                  «      | j
                  z  | j
                  dd¬«      | _        y )Né   r   ru  r\   r   ©r/   rf  )r1   r2   r~  r9   rd  rs  r   r8   rL  r^  r}  Úpsp_modulesrc  ÚlenÚpsp_bottleneckr0  Úlateral_convsÚ	fpn_convsÚappendÚfpn_bottleneck)r;   r,   rd  r<   s      €r)   r2   zBeitUperHead.__init__*  s…  ø€ Ü‰ÑÔà!×-Ñ-ˆÔØ"×.Ñ.Ð/°!Ñ3ˆÔØ×*Ñ*ˆŒÜŸ)™) D§M¡M°6×3DÑ3DÐRSÔTˆŒô 4Ø×ÑØ×Ñ˜RÑ Ø�M‰Mó
ˆÔô
 ,Ø×Ñ˜RÑ ¤3 t×'7Ñ'7Ó#8¸4¿=¹=Ñ#HÑHØ�M‰MØØô	
ˆÔô  Ÿ]™]›_ˆÔÜŸ™›ˆŒØ×+Ñ+¨C¨RÐ0ò 	iˆKØ×Ñ×%Ñ%¤m°KÀÇÁÐ\]Ô&^Ô_Ø�N‰N×!Ñ!¤-°·±¸t¿}¹}ÐZ[ÐefÔ"gÕhð	iô ,Ü�× Ñ Ó! D§M¡MÑ1Ø�M‰MØØô	
ˆÕr(   rÍ   c                 ó€   — |d   }t        j                  |g| j                  |«      ¢d¬«      }| j                  |«      S ru   )rG   rk   r‰  r‹  )r;   rÍ   r{  s      r)   Úpsp_forwardzBeitUperHead.psp_forwardL  sA   € Ø$ RÑ(ˆÜ—y‘y ,Ð!P°×1AÑ1AÀ,Ó1OÐ!PÐVWÔXˆØ×"Ñ" <Ó0Ð0r(   Úencoder_hidden_statesc                 ó6  — g }t        | j                  |«      D ]  \  }}|j                   ||«      «       Œ |j                  | j                  |«      «       t	        |«      }t        |dz
  dd«      D ]L  }||dz
     j                  dd  }||dz
     t        j                  j                  ||   |dd¬«      z   ||dz
  <   ŒN g }t        |dz
  «      D ])  }|j                   | j                  |   ||   «      «       Œ+ |j                  |d   «       t        |dz
  dd«      D ];  }t        j                  j                  ||   |d   j                  dd  dd¬«      ||<   Œ= t        j                  |d¬«      }| j                  |«      }	| j                  |	«      }	|	S )	Nr   r   r\   rA   rž   Fr^   rb   )ÚziprŒ  rŽ  r‘  rŠ  r-  rB   r   rh   ri   r�  rG   rk   r�  r^  )
r;   r’  ÚlateralsÚlateral_convr{  Úused_backbone_levelsr7  Ú
prev_shapeÚfpn_outsÚoutputs
             r)   rF   zBeitUperHead.forwardQ  s¹  € àˆÜ*-¨d×.@Ñ.@ÐBWÓ*Xò 	8Ñ&ˆL˜,Ø�O‰O™L¨Ó6Õ7ð	8ð 	�‰˜×(Ñ(Ð)>Ó?Ô@ô  # 8›}ÐÜÐ+¨aÑ/°°BÓ7ò 	ˆAØ! ! a¡%™×.Ñ.¨q¨rÐ2ˆJØ& q¨1¡u™o´·±×0IÑ0IØ˜‘ *°:ÈUð 1Jó 1ñ ˆH�Q˜‘UŠOð	ð ˆÜÐ+¨aÑ/Ó0ò 	<ˆAØ�O‰OÐ-˜DŸN™N¨1Ñ-¨h°q©kÓ:Õ;ð	<ð 	�‰˜ ™Ô%äÐ+¨aÑ/°°BÓ7ò 	ˆAÜŸ-™-×3Ñ3Ø˜‘ (¨1¡+×"3Ñ"3°A°BÐ"7¸jÐX]ð 4ó ˆH�QŠKð	ô —9‘9˜X¨1Ô-ˆØ×$Ñ$ XÓ.ˆØ—‘ Ó(ˆàˆr(   )r#   r$   r%   r&   r   r2   r‡   rG   r   r‘  rF   rH   rI   s   @r)   r…  r…  "  s\   ø„ ñð 
˜zð  
¨dõ  
ðD1¨¨e¯l©lÑ);ð 1ÀÇÁó 1ð
¨T°%·,±,Ñ-?ð ÀEÇLÁL÷ r(   r…  c                   óŠ   ‡ — e Zd ZdZ	 ddedededeeeef   z  ddf
ˆ fd„Zd	ee	j                     de	j                  fd
„Zˆ xZS )ÚBeitFCNHeadaÛ  
    Fully Convolution Networks for Semantic Segmentation. This head is implemented of
    [FCNNet](https://huggingface.co/papers/1411.4038>).

    Args:
        config (BeitConfig): Configuration.
        in_channels
        kernel_size (int): The kernel size for convs in the head. Default: 3.
        dilation (int): The dilation rate for convs in the head. Default: 1.


    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r,   Úin_indexr/   rg  r>   Nc           
      ó0  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        || _
        |dz  |z  }t        j                  «       | _        | j                  dkD  r—| j                  j                  t        | j                  | j
                  |||¬«      «       t!        | j                  dz
  «      D ]?  }| j                  j                  t        | j
                  | j
                  |||¬«      «       ŒA | j                  r8t        | j                  | j
                  z   | j
                  ||dz  ¬«      | _        t        j$                  | j
                  |j&                  d¬«      | _        y )NrA   r   )r/   rf  rg  r   rˆ  ru  )r1   r2   r9   rd  Úauxiliary_channelsrs  Úauxiliary_num_convsÚ	num_convsÚauxiliary_concat_inputÚconcat_inputr�  r   r0  ÚconvsrŽ  rc  r-  Úconv_catr8   rL  r^  )r;   r,   r�  r/   rg  Úconv_paddingry   r<   s          €r)   r2   zBeitFCNHead.__init__‚  sQ  ø€ ô 	‰ÑÔØ!×-Ñ-ˆÔØ×1Ñ1ˆŒØ×3Ñ3ˆŒØ"×9Ñ9ˆÔØ ˆŒà# qÑ(¨HÑ4ˆÜ—]‘]“_ˆŒ
Ø�>‰>˜AÒØ�J‰J×ÑÜØ×$Ñ$ d§m¡mÀÐVbÐmuôôô
 ˜4Ÿ>™>¨AÑ-Ó.ò 	�Ø—
‘
×!Ñ!Ü!ØŸ™ØŸ™Ø$/Ø ,Ø!)ôõð	ð ×ÒÜ)Ø× Ñ  4§=¡=Ñ0°$·-±-È[ÐbmÐqrÑbrôˆDŒMô Ÿ)™) D§M¡M°6×3DÑ3DÐRSÔTˆ�r(   r’  c                 óä   — || j                      }|}| j                  D ]
  } ||«      }Œ | j                  r(| j                  t	        j
                  ||gd¬«      «      }| j                  |«      }|S )Nr   rb   )r�  r¤  r£  r¥  rG   rk   r^  )r;   r’  r  rÍ   rx  s        r)   rF   zBeitFCNHead.forward¥  sn   € Ø(¨¯©Ñ7ˆØ ˆØ—J‘Jò 	0ˆDÙ  Ó/‰Mð	0à×ÒØ ŸM™M¬%¯)©)°X¸}Ð4MÐSTÔ*UÓVˆMØŸ™¨Ó6ˆØÐr(   )rA   r   r   )r#   r$   r%   r&   r   r€   r†   r2   r‡   rG   r   rF   rH   rI   s   @r)   rœ  rœ  s  su   ø„ ñð noñ!UØ ð!UØ,/ð!UØBEð!UØUXÐ[`ÐadÐfiÐaiÑ[jÑUjð!Uà	õ!UðF¨T°%·,±,Ñ-?ð ÀEÇLÁL÷ r(   rœ  c            	       ón   ‡ — e Zd ZdZd
dedededdfˆ fd„Zdej                  dej                  fd	„Zˆ xZ	S )ÚBeitFPNUpBlockuE   4x upsampling block: ConvTranspose â†’ BN â†’ GELU â†’ ConvTranspose.r9   r/   r0   r>   Nc                 ó   •— t         ‰| �  «        t        j                  ||||¬«      | _        t        j
                  |«      | _        t        j                  «       | _        t        j                  ||||¬«      | _	        y )Nr.   )
r1   r2   r   ÚConvTranspose2dÚconv_transpose1rl  rm  ÚGELUri  Úconv_transpose2)r;   r9   r/   r0   r<   s       €r)   r2   zBeitFPNUpBlock.__init__³  sb   ø€ Ü‰ÑÔÜ!×1Ñ1°+¸{ÐXcÐlrÔsˆÔÜŸ^™^¨KÓ8ˆÔÜŸ'™'›)ˆŒÜ!×1Ñ1°+¸{ÐXcÐlrÔsˆÕr(   rÍ   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S r   )r¬  rm  ri  r®  rá   s     r)   rF   zBeitFPNUpBlock.forwardº  sF   € Ø×,Ñ,¨]Ó;ˆØ×*Ñ*¨=Ó9ˆØŸ™¨Ó6ˆØ×,Ñ,¨]Ó;ˆØÐr(   )rA   rA   )
r#   r$   r%   r&   r€   r2   rG   r   rF   rH   rI   s   @r)   r©  r©  °  sH   ø„ ÙOñt Cð t°cð tÀsð tÐSWõ tð U§\¡\ð °e·l±l÷ r(   r©  c                   ót   ‡ — e Zd ZdZdefˆ fd„Zdeej                  df   deej                  df   fd„Z	ˆ xZ
S )ÚBeitFPNNeckz¤
    4-level feature pyramid neck for BeiT. Produces x4 upsample, x2 upsample,
    identity, and x2 downsample outputs from the four selected ViT feature maps.
    r,   c                 óò   •— t         ‰| �  «        t        |j                  «      | _        t        j                  |j                  |j                  dd¬«      | _        t        j                  dd¬«      | _	        y )NrA   r.   )
r1   r2   r©  r9   Úfpn1r   r«  Úfpn2Ú	MaxPool2dÚfpn4rÌ   s     €r)   r2   zBeitFPNNeck.__init__È  sX   ø€ Ü‰ÑÔÜ" 6×#5Ñ#5Ó6ˆŒ	Ü×&Ñ& v×'9Ñ'9¸6×;MÑ;MÐ[\ÐefÔgˆŒ	Ü—L‘L¨Q°qÔ9ˆ�	r(   Úfeature_maps.r>   c                 ó€   — | j                  |d   «      | j                  |d   «      |d   | j                  |d   «      fS r…   )r³  r´  r¶  )r;   r·  s     r)   rF   zBeitFPNNeck.forwardÎ  sC   € à�I‰I�l 1‘oÓ&Ø�I‰I�l 1‘oÓ&Ø˜‰OØ�I‰I�l 1‘oÓ&ð	
ð 	
r(   )r#   r$   r%   r&   r   r2   r†   rG   r   rF   rH   rI   s   @r)   r±  r±  Â  sD   ø„ ñð
:˜zõ :ð
 E¨%¯,©,¸Ð*;Ñ$<ð 
ÀÀuÇ|Á|ÐUXÐGXÑAY÷ 
r(   r±  c                   óª   ‡ — e Zd Zdeddfˆ fd„Zeee	 	 	 d
dej                  dz  dej                  dz  de
dee   deez  f
d	„«       «       «       Zˆ xZS )ÚBeitForSemanticSegmentationr,   r>   Nc                 ó`  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        | j                  j                  «      dk7  rt        d«      ‚t        |«      | _
        t        |«      | _        |j                  rt        |«      nd | _        | j!                  «        y )NFrK  r‡  zÇBeitForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.)r1   r2   rL  r(  r  rŠ  r,   Úout_indicesrC   r±  Úfpnr…  Údecode_headÚuse_auxiliary_headrœ  Úauxiliary_headr6  rÌ   s     €r)   r2   z$BeitForSemanticSegmentation.__init__Ù  s•   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜f¸Ô>ˆŒ	äˆt�{‰{×&Ñ&Ó'¨1Ò,Üð-óð ô
 ˜vÓ&ˆŒô (¨Ó/ˆÔØ5;×5NÒ5Nœk¨&Ô1ÐTXˆÔð 	�‰Õr(   r=   rQ  rr   r²   c                 óÀ  ‡‡‡‡— |�$| j                   j                  dk(  rt        d«      ‚ | j                  |fd|i|¤Ž}|j                  Š|j
                  \  Š}}}|| j                   j                  z  Š|| j                   j                  z  Št        ˆˆˆˆfd„| j                   j                  D «       «      }	| j                  |	«      }	| j                  |	«      }
d}| j                  �| j                  |	«      }d}|�>| j                  |
|| j                   j                  || j                   j                  ¬«      }t        ||
|j                  |j                   ¬«      S )aD  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, BeitForSemanticSegmentation
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")
        >>> model = BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than onerr   c              3   ó„   •K  — | ]7  }‰|d z
     dd…d d…f   j                  d d«      j                  ‰d‰‰«      –— Œ9 y­w)r   NrA   r\   )rE   rf   )Ú.0r7  rz   r’  Úpatch_heightÚpatch_widths     €€€€r)   ú	<genexpr>z6BeitForSemanticSegmentation.forward.<locals>.<genexpr>  sN   øè ø€ ò 
àð " ! a¡%Ñ(ª¨A©B¨Ñ/×9Ñ9¸!¸QÓ?×GÑGÈ
ÐTVÐXdÐfq×rñ
ùs   ƒ=A )Úignore_indexÚauxiliary_logitsÚauxiliary_loss_weightrS  )r,   rL  rC   r  rÍ   rB   r4   r†   r¼  r½  r¾  rÀ  r`  Úsemantic_loss_ignore_indexrÉ  r   r  )r;   r=   rQ  rr   r²   rW  ry   rY   rZ   r·  rU  rÈ  rT  rz   r’  rÄ  rÅ  s                @@@@r)   rF   z#BeitForSemanticSegmentation.forwardî  sh  û€ ðB Ð $§+¡+×"8Ñ"8¸AÒ"=ÜÐNÓOÐOØ�$—)‘)Øñ
à%=ð
ð ñ
ˆð !(× 5Ñ 5ÐØ'3×'9Ñ'9Ñ$ˆ
�A�v˜uØ §¡×!7Ñ!7Ñ7ˆØ˜tŸ{™{×5Ñ5Ñ5ˆô ö 
à—[‘[×,Ñ,ô
ó 
ˆð —x‘x Ó-ˆà×!Ñ! ,Ó/ˆàÐØ×ÑÐ*Ø#×2Ñ2°<Ó@ÐàˆØÐØ×%Ñ%ØØØ!Ÿ[™[×CÑCØ!1Ø&*§k¡k×&GÑ&Gð &ó ˆDô 'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   ra  )r#   r$   r%   r   r2   r   r   r   rG   r   r«   r   r   r†   r   rF   rH   rI   s   @r)   rº  rº  ×  sš   ø„ ð˜zð ¨dõ ð* Ø Øð -1Ø&*Ø).ñ	G
à—l‘l TÑ)ðG
ð —‘˜tÑ#ðG
ð #'ð	G
ð
 Ð+Ñ,ðG
ð 
Ð(Ñ	(òG
ó ó !ó ôG
r(   rº  zM
    BEiT backbone, to be used with frameworks like DETR and MaskFormer.
    c            	       óV   ‡ — e Zd Zˆ fd„Zeeededee	   de
fd„«       «       «       Zˆ xZS )ÚBeitBackbonec                 ó<  •— t         ‰| �  |«       t        |j                  dz   «      D �cg c]  }|j                  ‘Œ c}| _        t        |d¬«      | _        |j                  rt        |«      nt        j                  «       | _        | j                  «        y c c}w )Nr   FrK  )r1   r2   r-  r.  r9   Únum_featuresr(  r  Úadd_fpnr±  r   r  r½  r6  )r;   r,   ry   r<   s      €r)   r2   zBeitBackbone.__init__A  st   ø€ Ü‰Ñ˜Ô ä9>¸v×?WÑ?WÐZ[Ñ?[Ó9\Ö]°A˜V×/Ó/Ò]ˆÔÜ˜f¸Ô>ˆŒ	Ø*0¯.ª.”;˜vÔ&¼b¿k¹k»mˆŒð 	�‰Õùò ^s   «Br=   r²   r>   c                 ó*  — |j                   \  }}}}|| j                  j                  z  }|| j                  j                  z  } | j                  |fi |¤Ž}	|	j                  }
d}t        | j                  |
«      D ]d  \  }}|| j                  v sŒ| j                  j                  r4|dd…dd…dd…f   }|j                  dd«      }|j                  |d||«      }||fz  }Œf | j                  |«      }t        ||	j                  |	j                  ¬«      S )a:  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/beit-base-patch16-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 14, 14]
        ```r'   Nr   rA   r\   )r·  rÍ   r  )rB   r,   r4   r  rÍ   r”  Ústage_namesÚout_featuresÚreshape_hidden_statesrE   rf   r½  r   r  )r;   r=   r²   rz   ry   rY   rZ   rÄ  rÅ  rW  rÍ   r·  Ústager{  s                 r)   rF   zBeitBackbone.forwardK  s  € ð@ (4×'9Ñ'9Ñ$ˆ
�A�v˜uØ §¡×!7Ñ!7Ñ7ˆØ˜tŸ{™{×5Ñ5Ñ5ˆØ�$—)‘)˜LÑ3¨FÑ3ˆà×-Ñ-ˆØˆÜ#& t×'7Ñ'7¸Ó#Gò 	0ÑˆE�<Ø˜×)Ñ)Ò)Ø—;‘;×4Ò4Ø#/²°1±2²q°Ñ#9�LØ#/×#9Ñ#9¸!¸QÓ#?�LØ#/×#7Ñ#7¸
ÀBÈÐVaÓ#b�Là  Ñ/‘ð	0ð —x‘x Ó-ˆäØ%Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r(   )r#   r$   r%   r2   r   r   r   r   r   r   r   rF   rH   rI   s   @r)   rÌ  rÌ  ;  sN   ø„ ôð Ø Øð3
àð3
ð Ð+Ñ,ð3
ð 
ò	3
ó ó !ó ô3
r(   rÌ  )r\  rI  rº  r(  r  rÌ  )NrÏ   )FÚcollections.abcr   r   Údataclassesr   rG   r   r   Ú r	   r  Úactivationsr
   Úbackbone_utilsr   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_beitr   r"   ÚModuler+   rK   rƒ   ró   r½   r¿   rÙ   rä   rö   r  r(  r4  rI  r\  rc  rq  r}  r…  rœ  r©  r±  rº  rÌ  Ú__all__r'   r(   r)   ú<module>ræ     sÙ  ð÷* /Ý !ã ß å &Ý !ß HÝ 6Ý 9÷õ ÷ GÝ &Ý @ß BÑ Bß IÝ 5Ý *ñ ðôð
 ôÐ!;ó ó óðôH˜"Ÿ)™)ô Hô<S�R—Y‘Yô SôlV3˜rŸy™yô V3ð~ !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8-)�B—I‘Iô -)ô`ˆb�i‰iô ô %�2—9‘9ô %ô0<Ð*ô <ð~ ô"T˜/ó "Tó ð"TðJ ôJjÐ#ó Jjó ðJjôZ	v�—‘ô 	vñ ðôôS
Ð!4ó S
óðS
ñl ðôô/
Ð!4ó /
óð/
ôd�B—I‘Iô ôD
˜bŸi™iô 
ôS˜rŸy™yô Sô<N�2—9‘9ô Nôb:�"—)‘)ô :ôz�R—Y‘Yô ô$
�"—)‘)ô 
ð* ô`
Ð"5ó `
ó ð`
ñF ðôô
A
�=Ð"5ó A
óð
A
òH�r(   