Ë
    ÜÓ^jÐÚ  ã                   ó€  — d Z ddlZddlZddlmZ ddlZddlmZmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej@                  e!«      Z" ed¬«      e G d„ de«      «       «       Z# ed¬«      e G d„ de«      «       «       Z$ ed¬«      e G d„ de«      «       «       Z% ed¬«      e G d„ de«      «       «       Z&d„ Z'd„ Z( G d„ d ejR                  «      Z* G d!„ d"ejR                  «      Z+ G d#„ d$ejR                  «      Z, G d%„ d&ejR                  «      Z- G d'„ d(ejR                  «      Z. G d)„ d*ejR                  «      Z/ G d+„ d,ejR                  «      Z0 G d-„ d.ejR                  «      Z1 G d/„ d0ejR                  «      Z2 G d1„ d2ejR                  «      Z3 G d3„ d4e«      Z4 G d5„ d6ejR                  «      Z5e G d7„ d8e«      «       Z6e G d9„ d:e6«      «       Z7 ed;¬«       G d<„ d=e6«      «       Z8 ed>¬«       G d?„ d@e6«      «       Z9 edA¬«       G dB„ dCee6«      «       Z:g dD¢Z;y)Ez!PyTorch Swinv2 Transformer model.é    N)Ú	dataclass)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚBackboneMixinÚfilter_output_hidden_states)ÚGradientCheckpointingLayer)ÚBackboneOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleé   )ÚSwinv2ConfigzP
    Swinv2 encoder's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   óÐ   — e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   dZ
e	ej                  df   dz  ed<   dZe	ej                  df   dz  ed<   y)ÚSwinv2EncoderOutputaí  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlast_hidden_state.Úhidden_statesÚ
attentionsÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   Útupler   r   © ó    úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/swinv2/modeling_swinv2.pyr   r   (   s}   … ñð 37Ð�u×(Ñ(¨4Ñ/Ó6Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;ØCGÐ˜E %×"3Ñ"3°SÐ"8Ñ9¸DÑ@ÔGr%   r   zX
    Swinv2 model's outputs that also contains a pooling of the last hidden states.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚSwinv2ModelOutputa±  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   Úpooler_output.r   r   r   )r   r   r   r   r   r    r!   r"   r)   r   r#   r   r   r$   r%   r&   r(   r(   ?   s’   … ñ	ð 37Ð�u×(Ñ(¨4Ñ/Ó6Ø.2€M�5×$Ñ$ tÑ+Ó2Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;ØCGÐ˜E %×"3Ñ"3°SÐ"8Ñ9¸DÑ@ÔGr%   r(   z,
    Swinv2 masked image model outputs.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚSwinv2MaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NÚlossÚreconstruction.r   r   r   )r   r   r   r   r,   r    r!   r"   r-   r   r#   r   r   r$   r%   r&   r+   r+   Y   s‘   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø/3€N�E×%Ñ%¨Ñ,Ó3Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;ØCGÐ˜E %×"3Ñ"3°SÐ"8Ñ9¸DÑ@ÔGr%   r+   z2
    Swinv2 outputs for image classification.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚSwinv2ImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr,   Úlogits.r   r   r   )r   r   r   r   r,   r    r!   r"   r0   r   r#   r   r   r$   r%   r&   r/   r/   u   s‘   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;ØCGÐ˜E %×"3Ñ"3°SÐ"8Ñ9¸DÑ@ÔGr%   r/   c                 óÄ   — | j                   \  }}}}| j                  |||z  |||z  ||«      } | j                  dd«      j                  «       j                  d|||«      }|S )z2
    Partitions the given input into windows.
    é   r   éÿÿÿÿ©ÚshapeÚviewÚ	transposeÚ
contiguous)Úinput_featureÚwindow_sizeÚ
batch_sizeÚheightÚwidthÚnum_channelsÚwindowss          r&   Úwindow_partitionr@   ’   st   € ð /<×.AÑ.AÑ+€J�˜˜|Ø!×&Ñ&Ø�F˜kÑ)¨;¸ÀÑ8LÈkÐ[gó€Mð ×%Ñ% a¨Ó+×6Ñ6Ó8×=Ñ=¸bÀ+È{Ð\hÓi€GØ€Nr%   c                 óÀ   — | j                   d   }| j                  d||z  ||z  |||«      } | j                  dd«      j                  «       j                  d|||«      } | S )z?
    Merges windows to produce higher resolution features.
    r3   r2   r   r4   )r?   r:   r<   r=   r>   s        r&   Úwindow_reverserB   Ÿ   sg   € ð —=‘= Ñ$€LØ�l‰l˜2˜v¨Ñ4°e¸{Ñ6JÈKÐYdÐfrÓs€GØ×Ñ  1Ó%×0Ñ0Ó2×7Ñ7¸¸FÀEÈ<ÓX€GØ€Nr%   c            
       óÐ   ‡ — e Zd ZdZdˆ fd„	Zdej                  dededej                  fd„Z	 	 dd	ej                  dz  d
ej                  dz  dedeej                     fd„Zˆ xZS )ÚSwinv2EmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    c                 ó~  •— t         ‰| �  «        t        |«      | _        | j                  j                  }| j                  j
                  | _        |r4t        j                  t        j                  dd|j                  «      «      nd | _        |j                  r=t        j                  t        j                  d|dz   |j                  «      «      | _        nd | _        t        j                  |j                  «      | _        t        j"                  |j$                  «      | _        |j(                  | _        || _        y )Nr   )ÚsuperÚ__init__ÚSwinv2PatchEmbeddingsÚpatch_embeddingsÚnum_patchesÚ	grid_sizeÚ
patch_gridr   Ú	Parameterr    ÚzerosÚ	embed_dimÚ
mask_tokenÚuse_absolute_embeddingsÚposition_embeddingsÚ	LayerNormÚnormÚDropoutÚhidden_dropout_probÚdropoutÚ
patch_sizeÚconfig)ÚselfrY   Úuse_mask_tokenrJ   Ú	__class__s       €r&   rG   zSwinv2Embeddings.__init__¯   sâ   ø€ Ü‰ÑÔä 5°fÓ =ˆÔØ×+Ñ+×7Ñ7ˆØ×/Ñ/×9Ñ9ˆŒÙO]œ"Ÿ,™,¤u§{¡{°1°a¸×9IÑ9IÓ'JÔKÐcgˆŒà×)Ò)Ü')§|¡|´E·K±KÀÀ;ÐQRÁ?ÐTZ×TdÑTdÓ4eÓ'fˆDÕ$à'+ˆDÔ$ä—L‘L ×!1Ñ!1Ó2ˆŒ	Ü—z‘z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØˆ�r%   Ú
embeddingsr<   r=   Úreturnc                 ó¦  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }t        j                  j                  ||	|
fdd	¬
«      }|j                  dddd«      j                  dd|«      }t        j                  ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr3   ç      à?r   r   r2   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)r5   rR   r    ÚjitÚ
is_tracingrX   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolater6   Úcat)rZ   r]   r<   r=   rJ   Únum_positionsÚclass_pos_embedÚpatch_pos_embedrf   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r&   Úinterpolate_pos_encodingz)Swinv2Embeddings.interpolate_pos_encodingÂ   s`  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×2Ñ2²1°b°q°b°5Ñ9ˆØ×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr%   NÚpixel_valuesÚbool_masked_posrt   c                 óÂ  — |j                   \  }}}}| j                  |«      \  }}	| j                  |«      }|j                  «       \  }
}}|�K| j                  j                  |
|d«      }|j                  d«      j                  |«      }|d|z
  z  ||z  z   }| j                  �(|r|| j                  |||«      z   }n|| j                  z   }| j                  |«      }||	fS )Nr3   ç      ð?)r5   rI   rT   rb   rP   ÚexpandÚ	unsqueezeÚtype_asrR   rt   rW   )rZ   ru   rv   rt   Ú_r>   r<   r=   r]   Úoutput_dimensionsr;   Úseq_lenÚmask_tokensÚmasks                 r&   ÚforwardzSwinv2Embeddings.forwardê   sô   € ð *6×);Ñ);Ñ&ˆˆ<˜ Ø(,×(=Ñ(=¸lÓ(KÑ%ˆ
Ð%Ø—Y‘Y˜zÓ*ˆ
Ø!+§¡Ó!2Ñˆ
�G˜QàÐ&ØŸ/™/×0Ñ0°¸WÀbÓIˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJà×#Ñ#Ð/Ù'Ø'¨$×*GÑ*GÈ
ÐTZÐ\aÓ*bÑb‘
à'¨$×*BÑ*BÑB�
à—\‘\ *Ó-ˆ
àÐ,Ð,Ð,r%   ©F©NF)r   r   r   r   rG   r    r   Úintrt   r!   Ú
BoolTensorÚboolr#   r�   Ú__classcell__©r\   s   @r&   rD   rD   ª   s’   ø„ ñõð&&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]ió &DðV 48Ø).ñ	-à×'Ñ'¨$Ñ.ð-ð ×)Ñ)¨DÑ0ð-ð #'ð	-ð
 
ˆu�|‰|Ñ	÷-r%   rD   c                   óv   ‡ — e Zd ZdZˆ fd„Zd„ Zdej                  dz  deej                  ee
   f   fd„Zˆ xZS )rH   zì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 óæ  •— t         ‰| �  «        |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                  «      r|n||f}t        |t        j                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _
        |d   |d   z  |d   |d   z  f| _        t        j                  ||||¬«      | _        y )Nr   r   )Úkernel_sizeÚstride)rF   rG   Ú
image_sizerX   r>   rO   Ú
isinstanceÚcollectionsÚabcÚIterablerJ   rK   r   ÚConv2dÚ
projection)rZ   rY   r�   rX   r>   Úhidden_sizerJ   r\   s          €r&   rG   zSwinv2PatchEmbeddings.__init__  sì   ø€ Ü‰ÑÔØ!'×!2Ñ!2°F×4EÑ4E�Jˆ
Ø$*×$7Ñ$7¸×9IÑ9I�kˆÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ&ˆÔØ$ Q™-¨:°a©=Ñ8¸*ÀQ¹-È:ÐVWÉ=Ñ:XÐYˆŒäŸ)™) L°+È:Ð^hÔiˆ�r%   c                 ón  — || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|S )z9Pad pixel_values to be divisible by patch_size if needed.r   r   )rX   r   rk   Úpad)rZ   ru   r<   r=   Ú
pad_valuess        r&   Ú	maybe_padzSwinv2PatchEmbeddings.maybe_pad  s±   € à�4—?‘? 1Ñ%Ñ%¨Ò*Ø˜TŸ_™_¨QÑ/°%¸$¿/¹/È!Ñ:LÑ2LÑLÐMˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØ�D—O‘O AÑ&Ñ&¨!Ò+Ø˜Q  4§?¡?°1Ñ#5¸ÀÇÁÐQRÑASÑ8SÑ#SÐTˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØÐr%   ru   Nr^   c                 óà   — |j                   \  }}}}| j                  |||«      }| j                  |«      }|j                   \  }}}}||f}|j                  d«      j	                  dd«      }||fS )Nr2   r   )r5   r˜   r“   Úflattenr7   )rZ   ru   r|   r>   r<   r=   r]   r}   s           r&   r�   zSwinv2PatchEmbeddings.forward%  s}   € Ø)5×);Ñ);Ñ&ˆˆ<˜ à—~‘~ l°F¸EÓBˆØ—_‘_ \Ó2ˆ
Ø(×.Ñ.Ñˆˆ1ˆf�eØ# U˜OÐØ×'Ñ'¨Ó*×4Ñ4°Q¸Ó:ˆ
àÐ,Ð,Ð,r%   )r   r   r   r   rG   r˜   r    r!   r#   r   r„   r�   r‡   rˆ   s   @r&   rH   rH     sF   ø„ ñôjòð	- E×$5Ñ$5¸Ñ$<ð 	-ÀÀuÇ|Á|ÐUZÐ[^ÑU_ÐG_ÑA`÷ 	-r%   rH   c            	       ó²   ‡ — e Zd ZdZej
                  fdee   dedej                  ddfˆ fd„Z	d„ Z
d	ej                  d
eeef   dej                  fd„Zˆ xZS )ÚSwinv2PatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    Úinput_resolutionrf   Ú
norm_layerr^   Nc                 ó¤   •— t         ‰| �  «        || _        || _        t	        j
                  d|z  d|z  d¬«      | _         |d|z  «      | _        y )Né   r2   F©Úbias)rF   rG   r�   rf   r   ÚLinearÚ	reductionrT   )rZ   r�   rf   rž   r\   s       €r&   rG   zSwinv2PatchMerging.__init__>  sI   ø€ Ü‰ÑÔØ 0ˆÔØˆŒÜŸ™ 1 s¡7¨A°©G¸%Ô@ˆŒÙ˜q 3™wÓ'ˆ�	r%   c                 óŠ   — |dz  dk(  xs |dz  dk(  }|r.ddd|dz  d|dz  f}t         j                  j                  ||«      }|S )Nr2   r   r   )r   rk   r–   )rZ   r9   r<   r=   Ú
should_padr—   s         r&   r˜   zSwinv2PatchMerging.maybe_padE  sU   € Ø˜q‘j A‘oÒ:¨5°1©9¸©>ˆ
ÙØ˜Q  5¨1¡9¨a°¸!±Ð<ˆJÜŸM™M×-Ñ-¨m¸ZÓHˆMàÐr%   r9   Úinput_dimensionsc                 óº  — |\  }}|j                   \  }}}|j                  ||||«      }| j                  |||«      }|d d …dd d…dd d…d d …f   }|d d …dd d…dd d…d d …f   }	|d d …dd d…dd d…d d …f   }
|d d …dd d…dd d…d d …f   }t        j                  ||	|
|gd«      }|j                  |dd|z  «      }| j                  |«      }| j                  |«      }|S )Nr   r2   r   r3   r    )r5   r6   r˜   r    rm   r¤   rT   )rZ   r9   r§   r<   r=   r;   rf   r>   Úinput_feature_0Úinput_feature_1Úinput_feature_2Úinput_feature_3s               r&   r�   zSwinv2PatchMerging.forwardM  s  € Ø(‰ˆ�à(5×(;Ñ(;Ñ%ˆ
�C˜à%×*Ñ*¨:°v¸uÀlÓSˆàŸ™ }°f¸eÓDˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆäŸ	™	 ?°OÀ_ÐVeÐ"fÐhjÓkˆØ%×*Ñ*¨:°r¸1¸|Ñ;KÓLˆàŸ™ }Ó5ˆØŸ	™	 -Ó0ˆàÐr%   )r   r   r   r   r   rS   r#   r„   ÚModulerG   r˜   r    r   r�   r‡   rˆ   s   @r&   rœ   rœ   1  sr   ø„ ñ
ð XZ×WcÑWcñ (¨¨s©ð (¸#ð (È2Ï9É9ð (Ðhlõ (òð U§\¡\ð ÀUÈ3ÐPSÈ8Á_ð ÐY^×YeÑYe÷ r%   rœ   c            
       ó˜   ‡ — e Zd Zddgfˆ fd„	Z	 	 d
dej
                  dej                  dz  dedz  deej
                     fd„Z	d	„ Z
ˆ xZS )ÚSwinv2SelfAttentionr   c           
      ó‚  •— t         ‰| �  «        ||z  dk7  rt        d|› d|› d�«      ‚|| _        t	        ||z  «      | _        | j                  | j
                  z  | _        t        |t        j                  j                  «      r|n||f| _        || _        t        j                  t        j                   dt        j"                  |ddf«      z  «      «      | _        t        j&                  t        j(                  ddd	¬
«      t        j*                  d	¬«      t        j(                  d|d¬
«      «      | _        | j/                  «       \  }}| j1                  d|d¬«       | j1                  d|d¬«       t        j(                  | j                  | j                  |j2                  ¬
«      | _        t        j(                  | j                  | j                  d¬
«      | _        t        j(                  | j                  | j                  |j2                  ¬
«      | _        t        j:                  |j<                  «      | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)é
   r   r2   i   Tr¡   )ÚinplaceFÚrelative_coords_table)Ú
persistentÚrelative_position_index) rF   rG   Ú
ValueErrorÚnum_attention_headsr„   Úattention_head_sizeÚall_head_sizerŽ   r�   r�   r‘   r:   Úpretrained_window_sizer   rM   r    ÚlogÚonesÚlogit_scaleÚ
Sequentialr£   ÚReLUÚcontinuous_position_bias_mlpÚcreate_coords_table_and_indexÚregister_bufferÚqkv_biasÚqueryÚkeyÚvaluerU   Úattention_probs_dropout_probrW   )	rZ   rY   rf   Ú	num_headsr:   r»   r´   r¶   r\   s	           €r&   rG   zSwinv2SelfAttention.__init__h  sÔ  ø€ Ü‰ÑÔØ�‰?˜aÒÜØ# C 5Ð(^Ð_hÐ^iÐijÐkóð ð $-ˆÔ Ü#& s¨Y¡Ó#7ˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä% k´;·?±?×3KÑ3KÔL‰KÐS^Ð`kÐRlð 	Ôð '=ˆÔ#ÜŸ<™<¬¯	©	°"´u·z±zÀ9ÈaÐQRÐBSÓ7TÑ2TÓ(UÓVˆÔä,.¯M©MÜ�I‰I�a˜ 4Ô(¬"¯'©'¸$Ô*?ÄÇÁÈ3ÐPYÐ`eÔAfó-
ˆÔ)ð :>×9[Ñ9[Ó9]Ñ6ÐÐ6Ø×ÑÐ4Ð6KÐX]ÐÔ^Ø×ÑÐ6Ð8OÐ\aÐÔbä—Y‘Y˜t×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—9‘9˜T×/Ñ/°×1CÑ1CÈ%ÔPˆŒÜ—Y‘Y˜t×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—z‘z &×"EÑ"EÓFˆ�r%   Nr   Úattention_maskÚoutput_attentionsr^   c                 óR  — |j                   \  }}}| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }	t        j                  j                  |d¬«      t        j                  j                  |d¬«      j                  dd«      z  }
t        j                  | j                  t        j                  d«      ¬«      j!                  «       }|
|z  }
| j#                  | j$                  «      j                  d| j                  «      }|| j&                  j                  d«         j                  | j(                  d   | j(                  d   z  | j(                  d   | j(                  d   z  d«      }|j+                  ddd«      j-                  «       }d	t        j.                  |«      z  }|
|j1                  d«      z   }
|�“|j                   d   }|
j                  ||z  || j                  ||«      |j1                  d«      j1                  d«      z   }
|
|j1                  d«      j1                  d«      z   }
|
j                  d| j                  ||«      }
t        j                  j3                  |
d¬«      }| j5                  |«      }t        j6                  ||	«      }|j+                  dddd
«      j-                  «       }|j9                  «       d d | j:                  fz   }|j                  |«      }|r||f}|S |f}|S )Nr3   r   r2   re   éþÿÿÿg      Y@)Úmaxr   é   r   )r5   rÅ   r6   r¸   r¹   r7   rÆ   rÇ   r   rk   Ú	normalizer    Úclampr¾   Úmathr¼   ÚexprÁ   r´   r¶   r:   rj   r8   Úsigmoidrz   ÚsoftmaxrW   Úmatmulrb   rº   )rZ   r   rÊ   rË   r;   rf   r>   Úquery_layerÚ	key_layerÚvalue_layerÚattention_scoresr¾   Úrelative_position_bias_tableÚrelative_position_biasÚ
mask_shapeÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                      r&   r�   zSwinv2SelfAttention.forward…  sx  € ð )6×(;Ñ(;Ñ%ˆ
�C˜à�J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �H‰H�]Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ô Ÿ=™=×2Ñ2°;ÀBÐ2ÓGÌ"Ï-É-×JaÑJaØ˜2ð Kbó K
ç
‰)�B˜Ó
ñÐô —k‘k $×"2Ñ"2¼¿¹ÀÓ8LÔM×QÑQÓSˆØ+¨kÑ9ÐØ'+×'HÑ'HÈ×IcÑIcÓ'd×'iÑ'iØ�×(Ñ(ó(
Ð$ð ">¸d×>ZÑ>Z×>_Ñ>_Ð`bÓ>cÑ!d×!iÑ!iØ×Ñ˜QÑ $×"2Ñ"2°1Ñ"5Ñ5°t×7GÑ7GÈÑ7JÈT×M]ÑM]Ð^_ÑM`Ñ7`Ðbdó"
Ðð "8×!?Ñ!?ÀÀ1ÀaÓ!H×!SÑ!SÓ!UÐØ!#¤e§m¡mÐ4JÓ&KÑ!KÐØ+Ð.D×.NÑ.NÈqÓ.QÑQÐàÐ%à'×-Ñ-¨aÑ0ˆJØ/×4Ñ4Ø˜jÑ(¨*°d×6NÑ6NÐPSÐUXó à×(Ñ(¨Ó+×5Ñ5°aÓ8ñ 9Ðð  0°.×2JÑ2JÈ1Ó2M×2WÑ2WÐXYÓ2ZÑZÐØ/×4Ñ4°R¸×9QÑ9QÐSVÐX[Ó\Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆô Ÿ™ _°kÓBˆØ%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ð2ˆàˆð O\ÐM]ˆàˆr%   c                 ó8  — t        j                  | j                  d   dz
   | j                  d   t         j                  ¬«      j	                  «       }t        j                  | j                  d   dz
   | j                  d   t         j                  ¬«      j	                  «       }t        j
                  t        j                  ||gd¬«      «      j                  ddd«      j                  «       j                  d«      }| j                  d   dkD  rO|d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   n`| j                  d   dkD  rN|d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   |dz  }t        j                  |«      t        j                  t        j                  |«      dz   «      z  t        j                  d«      z  }|j                  t!        | j"                  j%                  «       «      j&                  «      }t        j                  | j                  d   «      }t        j                  | j                  d   «      }t        j
                  t        j                  ||gd¬«      «      }t        j(                  |d«      }|d d …d d …d f   |d d …d d d …f   z
  }|j                  ddd«      j                  «       }|d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   d| j                  d   z  dz
  z  cc<   |j+                  d	«      }	||	fS )
Nr   r   ©ÚdtypeÚij)Úindexingr2   é   rx   r3   )r    Úaranger:   Úint64ÚfloatÚstackÚmeshgridrj   r8   rz   r»   ÚsignÚlog2ÚabsrÒ   ÚtoÚnextrÁ   Ú
parametersrä   rš   Úsum)
rZ   Úrelative_coords_hÚrelative_coords_wr´   Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsr¶   s
             r&   rÂ   z1Swinv2SelfAttention.create_coords_table_and_indexÉ  s?  € ä!ŸL™L¨4×+;Ñ+;¸AÑ+>ÀÑ+BÐ)CÀT×EUÑEUÐVWÑEXÔ`e×`kÑ`kÔl×rÑrÓtÐÜ!ŸL™L¨4×+;Ñ+;¸AÑ+>ÀÑ+BÐ)CÀT×EUÑEUÐVWÑEXÔ`e×`kÑ`kÔl×rÑrÓtÐä�K‰KœŸ™Ð(9Ð;LÐ'MÐX\Ô]Ó^ß‰W�Q˜˜1Óß‰Z‹\ß‰Y�q‹\ð	 	ð ×&Ñ& qÑ)¨AÒ-Ø!¢!¢Qª¨1 *Ó-°×1LÑ1LÈQÑ1OÐRSÑ1SÑSÓ-Ø!¢!¢Qª¨1 *Ó-°×1LÑ1LÈQÑ1OÐRSÑ1SÑSÔ-Ø×Ñ˜aÑ  1Ò$Ø!¢!¢Qª¨1 *Ó-°×1AÑ1AÀ!Ñ1DÀqÑ1HÑHÓ-Ø!¢!¢Qª¨1 *Ó-°×1AÑ1AÀ!Ñ1DÀqÑ1HÑHÓ-Ø Ñ"Ðä�J‰JÐ,Ó-´·
±
¼5¿9¹9ÐEZÓ;[Ð^aÑ;aÓ0bÑbÔei×enÑenÐopÓeqÑqð 	ð !6× 8Ñ 8¼¸d×>_Ñ>_×>jÑ>jÓ>lÓ9m×9sÑ9sÓ tÐô —<‘< × 0Ñ 0°Ñ 3Ó4ˆÜ—<‘< × 0Ñ 0°Ñ 3Ó4ˆÜ—‘œUŸ^™^¨X°xÐ,@È4ÔPÓQˆÜŸ™ v¨qÓ1ˆØ(ªªA¨t¨Ñ4°~ÂaÈÊqÀjÑ7QÑQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  A¨×(8Ñ(8¸Ñ(;Ñ$;¸aÑ$?Ñ?Ó Ø"1×"5Ñ"5°bÓ"9Ðà$Ð&=Ð=Ð=r%   rƒ   )r   r   r   rG   r    r   r!   r†   r#   r�   rÂ   r‡   rˆ   s   @r&   r¯   r¯   g  sm   ø„ ØTUÐWXÐSYõ Gð@ 48Ø).ñ	Bà—|‘|ðBð ×)Ñ)¨DÑ0ðBð   $™;ð	Bð
 
ˆu�|‰|Ñ	óBöH#>r%   r¯   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚSwinv2SelfOutputc                 ó    •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |j                  «      | _        y ©N)rF   rG   r   r£   ÚdenserU   rÈ   rW   ©rZ   rY   rf   r\   s      €r&   rG   zSwinv2SelfOutput.__init__ñ  s6   ø€ Ü‰ÑÔÜ—Y‘Y˜s CÓ(ˆŒ
Ü—z‘z &×"EÑ"EÓFˆ�r%   r   Úinput_tensorr^   c                 óJ   — | j                  |«      }| j                  |«      }|S rþ   ©rÿ   rW   )rZ   r   r  s      r&   r�   zSwinv2SelfOutput.forwardö  s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆàÐr%   ©r   r   r   rG   r    r   r�   r‡   rˆ   s   @r&   rü   rü   ð  s2   ø„ ôGð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r%   rü   c            
       óŒ   ‡ — e Zd Zdˆ fd„	Z	 	 d	dej
                  dej                  dz  dedz  deej
                     fd„Z	ˆ xZ
S )
ÚSwinv2Attentionc           
      ó¾   •— t         ‰| �  «        t        ||||t        |t        j
                  j                  «      r|n||f¬«      | _        t        ||«      | _	        y )N©rY   rf   rÉ   r:   r»   )
rF   rG   r¯   rŽ   r�   r�   r‘   rZ   rü   Úoutput)rZ   rY   rf   rÉ   r:   r»   r\   s         €r&   rG   zSwinv2Attention.__init__þ  sY   ø€ Ü‰ÑÔÜ'ØØØØ#äÐ0´+·/±/×2JÑ2JÔKñ $:à(Ð*@ÐAô
ˆŒ	ô ' v¨sÓ3ˆ�r%   Nr   rÊ   rË   r^   c                 óh   — | j                  |||«      }| j                  |d   |«      }|f|dd  z   }|S )Nr   r   )rZ   r	  )rZ   r   rÊ   rË   Úself_outputsÚattention_outputrá   s          r&   r�   zSwinv2Attention.forward  sE   € ð —y‘y °Ð@QÓRˆØŸ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr%   ©r   rƒ   )r   r   r   rG   r    r   r!   r†   r#   r�   r‡   rˆ   s   @r&   r  r  ý  sW   ø„ õ4ð  48Ø).ñ		à—|‘|ð	ð ×)Ñ)¨DÑ0ð	ð   $™;ð		ð
 
ˆu�|‰|Ñ	÷	r%   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚSwinv2Intermediatec                 ó  •— t         ‰| �  «        t        j                  |t	        |j
                  |z  «      «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rþ   )rF   rG   r   r£   r„   Ú	mlp_ratiorÿ   rŽ   Ú
hidden_actÚstrr   Úintermediate_act_fnr   s      €r&   rG   zSwinv2Intermediate.__init__  sa   ø€ Ü‰ÑÔÜ—Y‘Y˜s¤C¨×(8Ñ(8¸3Ñ(>Ó$?Ó@ˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r%   r   r^   c                 óJ   — | j                  |«      }| j                  |«      }|S rþ   )rÿ   r  ©rZ   r   s     r&   r�   zSwinv2Intermediate.forward!  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr%   r  rˆ   s   @r&   r  r    s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r%   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚSwinv2Outputc                 óÌ   •— t         ‰| �  «        t        j                  t	        |j
                  |z  «      |«      | _        t        j                  |j                  «      | _	        y rþ   )
rF   rG   r   r£   r„   r  rÿ   rU   rV   rW   r   s      €r&   rG   zSwinv2Output.__init__)  sF   ø€ Ü‰ÑÔÜ—Y‘Yœs 6×#3Ñ#3°cÑ#9Ó:¸CÓ@ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r%   r   r^   c                 óJ   — | j                  |«      }| j                  |«      }|S rþ   r  r  s     r&   r�   zSwinv2Output.forward.  s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr%   r  rˆ   s   @r&   r  r  (  s#   ø„ ô>ð
 U§\¡\ð °e·l±l÷ r%   r  c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚSwinv2DropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr^   Nc                 ó0   •— t         ‰| �  «        || _        y rþ   )rF   rG   r  )rZ   r  r\   s     €r&   rG   zSwinv2DropPath.__init__<  s   ø€ Ü‰ÑÔØ"ˆ�r%   r   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )Nç        r   r   )r   )rä   Údevice)
r  Útrainingr5   Úndimr    Úrandrä   r!  ÚfloorÚdiv)rZ   r   Ú	keep_probr5   Úrandom_tensors        r&   r�   zSwinv2DropPath.forward@  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r%   c                 ó    — d| j                   › �S )Nzp=)r  ©rZ   s    r&   Ú
extra_reprzSwinv2DropPath.extra_reprI  s   € Ø�D—N‘NÐ#Ð$Ð$r%   )r   )r   r   r   r   rê   rG   r    r   r�   r  r+  r‡   rˆ   s   @r&   r  r  5  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r%   r  c                   óÊ   ‡ — e Zd Z	 dˆ fd„	Zdeeeef   eeef   f   fd„Zd„ Zd„ Z	 dde	j                  deeef   ded	z  dee	j                  e	j                  f   fd
„Zˆ xZS )ÚSwinv2Layerc           
      ón  •— t         ‰	| �  «        || _        | j                  |j                  |j                  f||f«      \  }}|d   | _        |d   | _        t        |||| j                  t        |t        j                  j                  «      r|n||f¬«      | _        t        j                  ||j                  ¬«      | _        |dkD  rt!        |«      nt        j"                  «       | _        t'        ||«      | _        t+        ||«      | _        t        j                  ||j                  ¬«      | _        y )Nr   r  ©Úepsr   )rF   rG   r�   Ú_compute_window_shiftr:   Ú
shift_sizer  rŽ   r�   r�   r‘   Ú	attentionr   rS   Úlayer_norm_epsÚlayernorm_beforer  ÚIdentityÚ	drop_pathr  Úintermediater  r	  Úlayernorm_after)
rZ   rY   rf   r�   rÉ   Údrop_path_rater2  r»   r:   r\   s
            €r&   rG   zSwinv2Layer.__init__N  s  ø€ ô 	‰ÑÔØ 0ˆÔØ"&×"<Ñ"<Ø×Ñ ×!3Ñ!3Ð4°zÀ:Ð6Nó#
Ñˆ�Zð ' q™>ˆÔØ$ Q™-ˆŒÜ(ØØØØ×(Ñ(äÐ0´+·/±/×2JÑ2JÔKñ $:à(Ð*@ÐAô
ˆŒô !#§¡¨S°f×6KÑ6KÔ LˆÔØ;IÈCÒ;Oœ¨Ô7ÔUW×U`ÑU`ÓUbˆŒÜ.¨v°sÓ;ˆÔÜ" 6¨3Ó/ˆŒÜ!Ÿ|™|¨C°V×5JÑ5JÔKˆÕr%   r^   c                 óî   — t        | j                  |«      D ��cg c]  \  }}t        ||«      ‘Œ }}}t        | j                  ||«      D ���cg c]  \  }}}||k  rdn|‘Œ }}}}||fS c c}}w c c}}}w ©Nr   )Úzipr�   Úmin)rZ   Útarget_window_sizeÚtarget_shift_sizeÚrÚwr:   Úsr2  s           r&   r1  z!Swinv2Layer._compute_window_shiftg  sy   € Ü-0°×1FÑ1FÐHZÓ-[×\¡T Q¨”s˜1˜a•yÐ\ˆÑ\Ü8;¸D×<QÑ<QÐS^Ð`qÓ8r×sÐs©W¨Q°°1˜1 š6‘a qÑ(Ðsˆ
ÒsØ˜JÐ&Ð&ùó ]ùÜss   šA*ÁA0c                 óä  — | j                   dk  ryt        j                  |«      }t        j                  |«      }||| j                  z
  k\  j	                  «       ||| j                   z
  k\  j	                  «       z   }||| j                  z
  k\  j	                  «       ||| j                   z
  k\  j	                  «       z   }|ddd…ddf   dz  |dddd…df   z   j                  |«      }t        || j                  «      }	|	j                  d| j                  | j                  z  «      }	|	j                  d«      |	j                  d«      z
  }
|
j                  |
dk7  d«      j                  |
dk(  d«      }
|
S )	uµ  Build the cyclic-shift attention mask for shifted-window MSA; returns None when shift_size is 0.

        Each (h, w) position belongs to one of 9 cyclic-shift regions (3 along each axis), encoded
        as ``h_region * 3 + w_region``. Regions per axis:
        - 0: indices ``[0, axis - window_size)``
        - 1: indices ``[axis - window_size, axis - shift_size)``
        - 2: indices ``[axis - shift_size, axis)``
        Implementation note: a single arithmetic pass on `torch.arange` (two comparisons +
        broadcast add) replaces the original 9-iteration nested-Python-loop slice-assignment â€”
        fully vectorised, no per-cell host-side scatter, no GPUâ†”host sync.
        r   Nr   r3   r   r2   g      YÀr   )
r2  r    rè   r:   Úlongrð   r@   r6   rz   Úmasked_fill)rZ   r<   r=   rä   Úh_idxÚw_idxÚh_regionÚw_regionÚimg_maskÚmask_windowsÚ	attn_masks              r&   Úget_attn_maskzSwinv2Layer.get_attn_maskl  se  € ð �?‰?˜aÒØÜ—‘˜VÓ$ˆÜ—‘˜UÓ#ˆØ˜V d×&6Ñ&6Ñ6Ñ6×<Ñ<Ó>À%È6ÐTX×TcÑTcÑKcÑBc×AiÑAiÓAkÑkˆØ˜U T×%5Ñ%5Ñ5Ñ5×;Ñ;Ó=ÀÈ%ÐRV×RaÑRaÑJaÑAa×@gÑ@gÓ@iÑiˆØ˜T¢1 d¨DÐ0Ñ1°AÑ5¸ÀÀtÊQÐPTÐATÑ8UÑU×YÑYÐZ_Ó`ˆÜ'¨°$×2BÑ2BÓCˆØ#×(Ñ(¨¨T×-=Ñ-=À×@PÑ@PÑ-PÓQˆØ ×*Ñ*¨1Ó-°×0FÑ0FÀqÓ0IÑIˆ	Ø×)Ñ)¨)°q©.¸&ÓA×MÑMÈiÐ[\ÉnÐ^aÓbˆ	ØÐr%   c                 óþ   — | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||«      }||fS r<  )r:   r   rk   r–   )rZ   r   r<   r=   Ú	pad_rightÚ
pad_bottomr—   s          r&   r˜   zSwinv2Layer.maybe_pad…  s�   € Ø×%Ñ%¨°×0@Ñ0@Ñ(@Ñ@ÀD×DTÑDTÑTˆ	Ø×&Ñ&¨°$×2BÑ2BÑ)BÑBÀd×FVÑFVÑVˆ
Ø˜˜A˜y¨!¨ZÐ8ˆ
ÜŸ™×)Ñ)¨-¸ÓDˆØ˜jÐ(Ð(r%   r   r§   rË   Nc                 óà  — |\  }}|j                  «       \  }}}|}	|j                  ||||«      }| j                  |||«      \  }}
|j                  \  }}}}| j                  dkD  r1t        j                  || j                   | j                   fd¬«      }n|}t        || j                  «      }|j                  d| j                  | j                  z  |«      }| j                  |||j                  ¬«      }|�|j                  |j                  «      }| j                  |||¬«      }|d   }|j                  d| j                  | j                  |«      }t        || j                  ||«      }| j                  dkD  r/t        j                  || j                  | j                  fd¬«      }n|}|
d   dkD  xs |
d   dkD  }|r|d d …d |…d |…d d …f   j                  «       }|j                  |||z  |«      }| j!                  |«      }|	| j#                  |«      z   }| j%                  |«      }| j'                  |«      }|| j#                  | j)                  |«      «      z   }|r	||d	   f}|S |f}|S )
Nr   )r   r2   )ÚshiftsÚdimsr3   rã   )rË   r   é   r   )rb   r6   r˜   r5   r2  r    Úrollr@   r:   rN  rä   rð   r!  r3  rB   r8   r5  r7  r8  r	  r9  )rZ   r   r§   rË   r<   r=   r;   r|   ÚchannelsÚshortcutr—   Ú
height_padÚ	width_padÚshifted_hidden_statesÚhidden_states_windowsrM  Úattention_outputsr  Úattention_windowsÚshifted_windowsÚ
was_paddedÚlayer_outputÚlayer_outputss                          r&   r�   zSwinv2Layer.forwardŒ  s™  € ð )‰ˆ�Ø"/×"4Ñ"4Ó"6Ñˆ
�A�xØ ˆð &×*Ñ*¨:°v¸uÀhÓOˆØ$(§N¡N°=À&È%Ó$PÑ!ˆ�zØ&3×&9Ñ&9Ñ#ˆˆ:�y !à�?‰?˜QÒÜ$)§J¡J¨}ÀtÇÁÐFVÐY]×YhÑYhÐXhÐEiÐpvÔ$wÑ!à$1Ð!ô !1Ð1FÈ×HXÑHXÓ YÐØ 5× :Ñ :¸2¸t×?OÑ?OÐRV×RbÑRbÑ?bÐdlÓ mÐØ×&Ñ& z°9ÀM×DWÑDWÐ&ÓXˆ	ØÐ Ø!Ÿ™Ð%:×%AÑ%AÓBˆIà ŸN™NÐ+@À)Ð_p˜NÓqÐà,¨QÑ/Ðà,×1Ñ1°"°d×6FÑ6FÈ×HXÑHXÐZbÓcÐÜ(Ð):¸D×<LÑ<LÈjÐZcÓdˆð �?‰?˜QÒÜ %§
¡
¨?ÀDÇOÁOÐUY×UdÑUdÐCeÐlrÔ sÑà /Ðà ‘] QÑ&Ò;¨*°Q©-¸!Ñ*;ˆ
ÙØ 1²!°W°f°W¸f¸u¸fÂaÐ2GÑ H× SÑ SÓ UÐà-×2Ñ2°:¸vÈ¹~ÈxÓXÐØ×-Ñ-Ð.?Ó@ˆØ  4§>¡>°-Ó#@Ñ@ˆà×(Ñ(¨Ó7ˆØ—{‘{ <Ó0ˆØ$ t§~¡~°d×6JÑ6JÈ<Ó6XÓ'YÑYˆá@Q˜Ð'8¸Ñ';Ð<ˆØÐð YeÐWfˆØÐr%   )r   r   r   r‚   )r   r   r   rG   r#   r„   r1  rN  r˜   r    r   r†   r�   r‡   rˆ   s   @r&   r-  r-  M  s˜   ø„ àqrõLð2'ÈeÐTYÐZ]Ð_bÐZbÑTcÐejÐknÐpsÐksÑetÐTtÑNuó 'ò
ò2)ð */ñ	5à—|‘|ð5ð    S ™/ð5ð   $™;ð	5ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷5r%   r-  c            
       ó|   ‡ — e Zd Z	 dˆ fd„	Z	 d	dej
                  deeef   dedz  deej
                     fd„Z	ˆ xZ
S )
ÚSwinv2Stagec	           
      ó|  •— t         ‰| �  «        || _        || _        g }	t	        |«      D ]?  }
t        ||||||
   |
dz  dk(  rdn|j                  dz  |¬«      }|	j                  |«       ŒA t        j                  |	«      | _
        |�& |||t        j                  ¬«      | _        d| _        y d | _        d| _        y )Nr2   r   )rY   rf   r�   rÉ   r:  r2  r»   )rf   rž   F)rF   rG   rY   rf   Úranger-  r:   Úappendr   Ú
ModuleListÚblocksrS   Ú
downsampleÚpointing)rZ   rY   rf   r�   ÚdepthrÉ   r7  rj  r»   ri  ÚiÚblockr\   s               €r&   rG   zSwinv2Stage.__init__Å  sÆ   ø€ ô 	‰ÑÔØˆŒØˆŒØˆÜ�u“ò 
	!ˆAÜØØØ!1Ø#Ø(¨™|Ø!" Q¡¨!¢™1°&×2DÑ2DÈÑ2IØ'=ôˆEð �M‰M˜%Õ ð
	!ô —m‘m FÓ+ˆŒð Ð!Ù(Ð)9¸sÌrÏ|É|Ô\ˆDŒOð ˆ�ð #ˆDŒOàˆ�r%   r   r§   rË   Nr^   c                 ó   — |\  }}t        | j                  «      D ]  \  }} ||||«      }|d   }Œ |}	| j                  �)|dz   dz  |dz   dz  }}
|||
|f}| j                  |	|«      }n||||f}||	|f}|r|dd  z  }|S )Nr   r   r2   )Ú	enumerateri  rj  )rZ   r   r§   rË   r<   r=   rm  Úlayer_modulerb  Ú!hidden_states_before_downsamplingÚheight_downsampledÚwidth_downsampledr}   Ústage_outputss                 r&   r�   zSwinv2Stage.forwardá  sÓ   € ð )‰ˆ�Ü(¨¯©Ó5ò 	-‰OˆAˆ|Ù(ØØ Ø!óˆMð *¨!Ñ,‰Mð	-ð -:Ð)Ø�?‰?Ð&Ø5;¸a±ZÀAÑ4EÈÐPQÉ	ÐVWÑGWÐ 1ÐØ!'¨Ð0BÐDUÐ VÐØ ŸO™OÐ,MÐO_Ó`‰Mà!'¨°¸Ð >Ðà&Ð(IÐK\Ð]ˆáØ˜]¨1¨2Ð.Ñ.ˆMØÐr%   r  r‚   )r   r   r   rG   r    r   r#   r„   r†   r�   r‡   rˆ   s   @r&   rd  rd  Ä  sU   ø„ àmnõð@ */ñ	à—|‘|ðð    S ™/ðð   $™;ð	ð
 
ˆu�|‰|Ñ	÷r%   rd  c                   óŠ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  deeef   dedz  dedz  dedz  dedz  d	ee	z  fd
„Z
ˆ xZS )ÚSwinv2Encoderc                 ó>  •— t         ‰	| �  «        t        |j                  «      | _        || _        | j
                  j                  �|j                  }t        j                  d|j                  t        |j                  «      d¬«      D �cg c]  }|j                  «       ‘Œ }}g }t        | j                  «      D ]º  }t        |t        |j                  d|z  z  «      |d   d|z  z  |d   d|z  z  f|j                  |   |j                   |   |t        |j                  d | «      t        |j                  d |dz    «       || j                  dz
  k  rt"        nd ||   ¬«      }|j%                  |«       Œ¼ t'        j(                  |«      | _        d| _        y c c}w )Nr   Úcpu)r!  r2   r   )rY   rf   r�   rl  rÉ   r7  rj  r»   F)rF   rG   ÚlenÚdepthsÚ
num_layersrY   Úpretrained_window_sizesr    Úlinspacer:  ró   Úitemrf  rd  r„   rO   rÉ   rœ   rg  r   rh  ÚlayersÚgradient_checkpointing)
rZ   rY   rK   r}  ÚxÚdprr€  Úi_layerÚstager\   s
            €r&   rG   zSwinv2Encoder.__init__  sw  ø€ Ü‰ÑÔÜ˜fŸm™mÓ,ˆŒØˆŒØ�;‰;×.Ñ.Ð:Ø&,×&DÑ&DÐ#Ü!&§¡°°6×3HÑ3HÌ#ÈfÏmÉmÓJ\ÐejÔ!kÖl˜Aˆq�v‰v�xÐlˆÐlàˆÜ˜TŸ_™_Ó-ò 	!ˆGÜØÜ˜×(Ñ(¨1¨g©:Ñ5Ó6Ø"+¨A¡,°1°g±:Ñ">À	È!ÁÐQRÐT[ÑQ[Ñ@\Ð!]Ø—m‘m GÑ,Ø ×*Ñ*¨7Ñ3Øœc &§-¡-°°Ð"9Ó:¼SÀÇÁÈ}ÐQXÐ[\ÑQ\ÐA]Ó=^Ð_Ø29¸D¿O¹OÈaÑ<OÒ2OÕ-ÐVZØ'>¸wÑ'Gô	ˆEð �M‰M˜%Õ ð	!ô —m‘m FÓ+ˆŒà&+ˆÕ#ùò# ms   ÂFr   r§   rË   NÚoutput_hidden_statesÚ(output_hidden_states_before_downsamplingÚreturn_dictr^   c                 óÎ  — |rdnd }|rdnd }|rdnd }	|rE|j                   \  }
}} |j                  |
g|¢|‘­Ž }|j                  dddd«      }||fz  }||fz  }t        | j                  «      D ]Î  \  }} ||||«      }|d   }|d   }|d   }|d   |d   f}|rP|rN|j                   \  }
}} |j                  |
g|d   |d   f¢|‘­Ž }|j                  dddd«      }||fz  }||fz  }nI|rG|sE|j                   \  }
}} |j                  |
g|¢|‘­Ž }|j                  dddd«      }||fz  }||fz  }|sŒÇ|	|dd  z  }	ŒÐ |st        d„ |||	|fD «       «      S t        |||	|¬	«      S )
Nr$   r   r   r   r2   rÍ   r3   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrþ   r$   )Ú.0Úvs     r&   ú	<genexpr>z(Swinv2Encoder.forward.<locals>.<genexpr>R  s   è ø€ ò àØ�=ô ñùs   ‚)r   r   r   r   )r5   r6   rj   rp  r€  r#   r   )rZ   r   r§   rË   r†  r‡  rˆ  Úall_hidden_statesÚall_reshaped_hidden_statesÚall_self_attentionsr;   r|   r”   Úreshaped_hidden_staterm  rq  rb  rr  r}   s                      r&   r�   zSwinv2Encoder.forward  sB  € ñ #7™B¸DÐÙ+?¡RÀTÐ"Ù$5™b¸4ÐáØ)6×)<Ñ)<Ñ&ˆJ˜˜;à$6 M×$6Ñ$6°zÐ$bÐDTÐ$bÐVaÒ$bÐ!Ø$9×$AÑ$AÀ!ÀQÈÈ1Ó$MÐ!Ø -Ð!1Ñ1ÐØ&Ð+@Ð*BÑBÐ&ä(¨¯©Ó5ò  	9‰OˆAˆ|Ù(ØØ Ø!óˆMð *¨!Ñ,ˆMØ0=¸aÑ0@Ð-Ø -¨aÑ 0Ðà 1°"Ñ 5Ð7HÈÑ7LÐMÐá#Ñ(PØ-N×-TÑ-TÑ*�
˜A˜{ð )OÐ(I×(NÑ(NØð)Ø"3°AÑ"6Ð8IÈ!Ñ8LÐ!Mð)ØOZò)Ð%ð )>×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø!Ð&GÐ%IÑIÐ!Ø*Ð/DÐ.FÑFÑ*Ù%Ñ.VØ-:×-@Ñ-@Ñ*�
˜A˜{à(:¨×(:Ñ(:¸:Ð(fÐHXÐ(fÐZeÒ(fÐ%Ø(=×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø! mÐ%5Ñ5Ð!Ø*Ð/DÐ.FÑFÐ*â Ø# }°Q°RÐ'8Ñ8Ñ#ðA 	9ñD Üñ à'Ð):Ð<OÐQkÐlôó ð ô #Ø+Ø+Ø*Ø#=ô	
ð 	
r%   ))r   r   r   r   )FFFT)r   r   r   rG   r    r   r#   r„   r†   r   r�   r‡   rˆ   s   @r&   rw  rw     s‰   ø„ õ,ð: */Ø,1Ø@EØ#'ñC
à—|‘|ðC
ð    S ™/ðC
ð   $™;ð	C
ð
 # T™kðC
ð 37¸±+ðC
ð ˜D‘[ðC
ð 
Ð$Ñ	$÷C
r%   rw  c                   óh   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	 e
j                  «       ˆ fd„«       Zˆ xZS )ÚSwinv2PreTrainedModelrY   Úswinv2ru   )ÚimageTrd  c                 ó"  •— t         ‰| �  |«       t        |t        «      rX|j                  �t        j                  |j                  «       |j                  � t        j                  |j                  «       yyt        |t        «      r‡t        j                  |j                  t        j                  d«      «       |j                  «       \  }}t        j                  |j                  |«       t        j                  |j                   |«       yy)zInitialize the weightsNr²   )rF   Ú_init_weightsrŽ   rD   rP   ÚinitÚzeros_rR   r¯   Ú	constant_r¾   rÒ   r¼   rÂ   Úcopy_r´   r¶   )rZ   Úmoduler´   r¶   r\   s       €r&   r—  z#Swinv2PreTrainedModel._init_weightsi  sÉ   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ.Ô/Ø× Ñ Ð,Ü—‘˜F×-Ñ-Ô.Ø×)Ñ)Ð5Ü—‘˜F×6Ñ6Õ7ð 6ä˜Ô 3Ô4Ü�N‰N˜6×-Ñ-¬t¯x©x¸«|Ô<Ø=C×=aÑ=aÓ=cÑ:Ð!Ð#:Ü�J‰J�v×3Ñ3Ð5JÔKÜ�J‰J�v×5Ñ5Ð7NÕOð	 5r%   )r   r   r   r   r"   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesr    Úno_gradr—  r‡   rˆ   s   @r&   r“  r“  `  sD   ø… àÓØ ÐØ$€OØ!ÐØ&*Ð#Ø&˜Ðà€U‡]�]ƒ_óPó ôPr%   r“  c                   ó®   ‡ — e Zd Zdˆ fd„	Zd„ Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  de	d	e	dz  d
e
ez  fd„«       Zˆ xZS )ÚSwinv2Modelc                 óê  •— t         ‰| �  |«       || _        t        |j                  «      | _        t        |j                  d| j
                  dz
  z  z  «      | _        t        ||¬«      | _
        t        || j                  j                  «      | _        t        j                  | j                  |j                   ¬«      | _        |rt        j$                  d«      nd| _        | j)                  «        y)a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r2   r   )r[   r/  N)rF   rG   rY   rz  r{  r|  r„   rO   Únum_featuresrD   r]   rw  rL   Úencoderr   rS   r4  Ú	layernormÚAdaptiveAvgPool1dÚpoolerÚ	post_init)rZ   rY   Úadd_pooling_layerr[   r\   s       €r&   rG   zSwinv2Model.__init__|  sµ   ø€ ô 	‰Ñ˜Ô ØˆŒÜ˜fŸm™mÓ,ˆŒÜ × 0Ñ 0°1¸¿¹È1Ñ9LÑ3MÑ MÓNˆÔä*¨6À.ÔQˆŒÜ$ V¨T¯_©_×-GÑ-GÓHˆŒäŸ™ d×&7Ñ&7¸V×=RÑ=RÔSˆŒÙ1B”b×*Ñ*¨1Ô-ÈˆŒð 	�‰Õr%   c                 ó.   — | j                   j                  S rþ   ©r]   rI   r*  s    r&   Úget_input_embeddingsz Swinv2Model.get_input_embeddings‘  ó   € Ø�‰×/Ñ/Ð/r%   Nru   rv   rË   r†  rt   rˆ  r^   c                 ó@  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        d«      ‚| j                  |||¬«      \  }}	| j                  ||	|||¬«      }
|
d   }| j                  |«      }d}| j                  �7| j                  |j                  dd«      «      }t        j                  |d«      }|s||f|
dd z   }|S t        |||
j                  |
j                  |
j                  ¬«      S )	zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rv   rt   )rË   r†  rˆ  r   r   r2   )r   r)   r   r   r   )rY   rË   r†  rˆ  r·   r]   r§  r¨  rª  r7   r    rš   r(   r   r   r   )rZ   ru   rv   rË   r†  rt   rˆ  ÚkwargsÚembedding_outputr§   Úencoder_outputsÚsequence_outputÚpooled_outputr	  s                 r&   r�   zSwinv2Model.forward”  sJ  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐÜÐ?Ó@Ð@à-1¯_©_Ø¨/ÐTlð .=ó .
Ñ*ÐÐ*ð Ÿ,™,ØØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØŸ.™.¨Ó9ˆàˆØ�;‰;Ð"Ø ŸK™K¨×(AÑ(AÀ!ÀQÓ(GÓHˆMÜ!ŸM™M¨-¸Ó;ˆMáØ% }Ð5¸ÈÈÐ8KÑKˆFàˆMä Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1Ø#2×#IÑ#Iô
ð 	
r%   )TF©NNNNFN)r   r   r   rG   r¯  r   r    r!   r…   r†   r#   r(   r�   r‡   rˆ   s   @r&   r¤  r¤  y  s¡   ø„ õò*0ð ð 26Ø37Ø)-Ø,0Ø).Ø#'ñ6
à×'Ñ'¨$Ñ.ð6
ð ×)Ñ)¨DÑ0ð6
ð   $™;ð	6
ð
 # T™kð6
ð #'ð6
ð ˜D‘[ð6
ð 
Ð"Ñ	"ò6
ó ô6
r%   r¤  a~  
        Swinv2 Model with a decoder on top for masked image modeling, as proposed in
    [SimMIM](https://huggingface.co/papers/2111.09886).

        <Tip>

        Note that we provide a script to pre-train this model on custom data in our [examples
        directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

        </Tip>
    c                   ó¦   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
„«       Zˆ xZS )ÚSwinv2ForMaskedImageModelingc                 óŠ  •— t         ‰| �  |«       t        |dd¬«      | _        t	        |j
                  d|j                  dz
  z  z  «      }t        j                  t        j                  ||j                  dz  |j                  z  d¬«      t        j                  |j                  «      «      | _        | j                  «        y )NFT)r¬  r[   r2   r   )Úin_channelsÚout_channelsr‹   )rF   rG   r¤  r”  r„   rO   r|  r   r¿   r’   Úencoder_strider>   ÚPixelShuffleÚdecoderr«  )rZ   rY   r¦  r\   s      €r&   rG   z%Swinv2ForMaskedImageModeling.__init__Ý  s¡   ø€ Ü‰Ñ˜Ô ä! &¸EÐRVÔWˆŒä˜6×+Ñ+¨a°F×4EÑ4EÈÑ4IÑ.JÑJÓKˆÜ—}‘}Ü�I‰IØ(°v×7LÑ7LÈaÑ7OÐRX×ReÑReÑ7eÐstôô �O‰O˜F×1Ñ1Ó2ó	
ˆŒð 	�‰Õr%   Nru   rv   rË   r†  rt   rˆ  r^   c                 óÎ  — |�|n| j                   j                  }| j                  ||||||¬«      }|d   }	|	j                  dd«      }	|	j                  \  }
}}t        j                  |dz  «      x}}|	j                  |
|||«      }	| j                  |	«      }d}|��| j                   j                  | j                   j                  z  }|j                  d||«      }|j                  | j                   j                  d«      j                  | j                   j                  d«      j                  d«      j                  «       }t        j                  j!                  ||d¬	«      }||z  j#                  «       |j#                  «       d
z   z  | j                   j$                  z  }|s|f|dd z   }|�|f|z   S |S t'        |||j(                  |j*                  |j,                  ¬«      S )a�  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, Swinv2ForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = Swinv2ForMaskedImageModeling.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 256, 256]
        ```N)rv   rË   r†  rt   rˆ  r   r   r2   r`   r3   Únone)r¤   gñhãˆµøä>)r,   r-   r   r   r   )rY   rˆ  r”  r7   r5   rÒ   r%  ri   r¿  r�   rX   Úrepeat_interleaverz   r8   r   rk   Úl1_lossró   r>   r+   r   r   r   )rZ   ru   rv   rË   r†  rt   rˆ  r²  rá   rµ  r;   r>   Úsequence_lengthr<   r=   Úreconstructed_pixel_valuesÚmasked_im_lossrb   r€   Úreconstruction_lossr	  s                        r&   r�   z$Swinv2ForMaskedImageModeling.forwardí  så  € ðP &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ+Ø/Ø!5Ø%=Ø#ð ó 
ˆð " !™*ˆà)×3Ñ3°A°qÓ9ˆØ4C×4IÑ4IÑ1ˆ
�L /ÜŸ™ O°SÑ$8Ó9Ð9ˆ�Ø)×1Ñ1°*¸lÈFÐTYÓZˆð &*§\¡\°/Ó%BÐ"àˆØÑ&Ø—;‘;×)Ñ)¨T¯[©[×-CÑ-CÑCˆDØ-×5Ñ5°b¸$ÀÓEˆOà×1Ñ1°$·+±+×2HÑ2HÈ!ÓLß"Ñ" 4§;¡;×#9Ñ#9¸1Ó=ß‘˜1“ß‘“ð	 ô #%§-¡-×"7Ñ"7¸ÐF`ÐlrÐ"7Ó"sÐØ1°DÑ8×=Ñ=Ó?À4Ç8Á8Ã:ÐPTÑCTÑUÐX\×XcÑXc×XpÑXpÑpˆNáØ0Ð2°W¸Q¸R°[Ñ@ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYä.ØØ5Ø!×/Ñ/Ø×)Ñ)Ø#*×#AÑ#Aô
ð 	
r%   r·  )r   r   r   rG   r   r    r!   r…   r†   r#   r+   r�   r‡   rˆ   s   @r&   r¹  r¹  Î  s¥   ø„ ôð  ð 26Ø37Ø)-Ø,0Ø).Ø#'ñS
à×'Ñ'¨$Ñ.ðS
ð ×)Ñ)¨DÑ0ðS
ð   $™;ð	S
ð
 # T™kðS
ð #'ðS
ð ˜D‘[ðS
ð 
Ð0Ñ	0òS
ó ôS
r%   r¹  aæ  
    Swinv2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune SwinV2 on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   ó¦   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
„«       Zˆ xZS )ÚSwinv2ForImageClassificationc                 ó>  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j                  dkD  r4t        j                  | j                  j                  |j                  «      nt        j                  «       | _	        | j                  «        y r<  )rF   rG   Ú
num_labelsr¤  r”  r   r£   r¦  r6  Ú
classifierr«  )rZ   rY   r\   s     €r&   rG   z%Swinv2ForImageClassification.__init__T  sx   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ! &Ó)ˆŒð GM×FWÑFWÐZ[ÒF[ŒB�I‰I�d—k‘k×.Ñ.°×0AÑ0AÔBÔac×alÑalÓanð 	Œð
 	�‰Õr%   Nru   ÚlabelsrË   r†  rt   rˆ  r^   c                 óV  — |�|n| j                   j                  }| j                  |||||¬«      }|d   }	| j                  |	«      }
d}|�| j	                  ||
| j                   «      }|s|
f|dd z   }|�|f|z   S |S t        ||
|j                  |j                  |j                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)rË   r†  rt   rˆ  r   r2   )r,   r0   r   r   r   )	rY   rˆ  r”  rÌ  Úloss_functionr/   r   r   r   )rZ   ru   rÍ  rË   r†  rt   rˆ  r²  rá   r¶  r0   r,   r	  s                r&   r�   z$Swinv2ForImageClassification.forwardb  sÒ   € ð" &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ/Ø!5Ø%=Ø#ð ó 
ˆð   ™
ˆà—‘ Ó/ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä*ØØØ!×/Ñ/Ø×)Ñ)Ø#*×#AÑ#Aô
ð 	
r%   r·  )r   r   r   rG   r   r    r!   Ú
LongTensorr†   r#   r/   r�   r‡   rˆ   s   @r&   rÉ  rÉ  D  sœ   ø„ ô ð ð 26Ø*.Ø)-Ø,0Ø).Ø#'ñ,
à×'Ñ'¨$Ñ.ð,
ð × Ñ  4Ñ'ð,
ð   $™;ð	,
ð
 # T™kð,
ð #'ð,
ð ˜D‘[ð,
ð 
Ð,Ñ	,ò,
ó ô,
r%   rÉ  zO
    Swinv2 backbone, to be used with frameworks like DETR and MaskFormer.
    c                   óx   ‡ — e Zd Zˆ fd„Zd„ Zeee	 	 	 d
dede	dz  de	dz  de	dz  de
f
d	„«       «       «       Zˆ xZS )ÚSwinv2Backbonec           	      óh  •— t         ‰| �  |«       |j                  gt        t	        |j
                  «      «      D �cg c]  }t        |j                  d|z  z  «      ‘Œ c}z   | _        t        |«      | _	        t        || j                  j                  «      | _        | j                  «        y c c}w )Nr2   )rF   rG   rO   rf  rz  r{  r„   r¦  rD   r]   rw  rL   r§  r«  )rZ   rY   rm  r\   s      €r&   rG   zSwinv2Backbone.__init__˜  s�   ø€ Ü‰Ñ˜Ô à#×-Ñ-Ð.ÔX]Ô^aÐbh×boÑboÓ^pÓXqÖ1rÐST´#°f×6FÑ6FÈÈAÉÑ6MÕ2NÒ1rÑrˆÔÜ*¨6Ó2ˆŒÜ$ V¨T¯_©_×-GÑ-GÓHˆŒð 	�‰Õùò 2ss   ½"B/c                 ó.   — | j                   j                  S rþ   r®  r*  s    r&   r¯  z#Swinv2Backbone.get_input_embeddings¢  r°  r%   Nru   rË   r†  rˆ  r^   c                 ó
  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      \  }}| j                  |||dd|¬«      }|r|j                  n|d   }	d}
t        | j                  |	«      D ]  \  }}|| j                  v sŒ|
|fz  }
Œ |s|
f}|r	||d   fz  }|r	||d   fz  }|S t        |
|r|j                  nd|j                  ¬«      S )	aœ  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swinv2-tiny-patch4-window8-256", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 2048, 7, 7]
        ```NT)rË   r†  r‡  rˆ  r3   r$   r   r2   )Úfeature_mapsr   r   )rY   rˆ  r†  rË   r]   r§  r   r=  Ústage_namesÚout_featuresr   r   r   )rZ   ru   rË   r†  rˆ  r²  r³  r§   rá   r   rÖ  r…  Úhidden_stater	  s                 r&   r�   zSwinv2Backbone.forward¥  sA  € ðJ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà-1¯_©_¸\Ó-JÑ*ÐÐ*à—,‘,ØØØ/Ø!%Ø59Ø#ð ó 
ˆñ ;F˜×6Ò6È7ÐSUÉ;ˆàˆÜ#& t×'7Ñ'7¸Ó#Gò 	0ÑˆE�<Ø˜×)Ñ)Ò)Ø  Ñ/‘ð	0ñ Ø"�_ˆFÙ#Ø˜7 1™:˜-Ñ'�Ù Ø˜7 1™:˜-Ñ'�ØˆMäØ%Ù3G˜'×/Ò/ÈTØ×)Ñ)ô
ð 	
r%   )NNN)r   r   r   rG   r¯  r   r
   r   r   r†   r   r�   r‡   rˆ   s   @r&   rÒ  rÒ  ’  s   ø„ ôò0ð Ø Øð *.Ø,0Ø#'ñF
àðF
ð   $™;ðF
ð # T™kð	F
ð
 ˜D‘[ðF
ð 
òF
ó ó !ó ôF
r%   rÒ  )rÉ  r¹  r¤  r“  rÒ  )<r   Úcollections.abcr�   rÒ   Údataclassesr   r    r   r   Ú r   r˜  Úactivationsr   Úbackbone_utilsr	   r
   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úconfiguration_swinv2r   Ú
get_loggerr   Úloggerr   r(   r+   r/   r@   rB   r­   rD   rH   rœ   r¯   rü   r  r  r  r  r-  rd  rw  r“  r¤  r¹  rÉ  rÒ  Ú__all__r$   r%   r&   ú<module>rè     s«  ðñ (ã Û Ý !ã ß å &Ý !ß HÝ 9Ý .Ý -ß DÓ DÝ -Ý .ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ôH˜+ó Hó óðHñ  ðôð
 ôH˜ó Hó óðHñ& ðôð
 ôH kó Hó óðHñ* ðôð
 ôH +ó Hó óðHò,	òôY-�r—y‘yô Y-ôz'-˜BŸI™Iô '-ôT3˜Ÿ™ô 3ôlE>˜"Ÿ)™)ô E>ôR
�r—y‘yô 
ô�b—i‘iô ô6˜Ÿ™ô ô 	�2—9‘9ô 	ô%�R—Y‘Yô %ô0t�"—)‘)ô tôn9Ð,ô 9ôx]
�B—I‘Iô ]
ð@ ôP˜Oó Pó ðPð0 ôP
Ð'ó P
ó ðP
ñf ð
ôôe
Ð#8ó e
óðe
ñP ðôô<
Ð#8ó <
óð<
ñ~ ðôô
W
�]Ð$9ó W
óð
W
òt�r%   