
    ^j                        d Z ddlZddlZddlmZ ddlZddlmZmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej@                  e!      Z" ed      e G d de                    Z# ed      e G d de                    Z$ ed      e G d de                    Z% ed      e G d de                    Z&d Z'd Z( G d d ejR                        Z* G d! d"ejR                        Z+ G d# d$ejR                        Z, G d% d&ejR                        Z- G d' d(ejR                        Z. G d) d*ejR                        Z/ G d+ d,ejR                        Z0 G d- d.ejR                        Z1 G d/ d0ejR                        Z2 G d1 d2ejR                        Z3 G d3 d4e      Z4 G d5 d6ejR                        Z5e G d7 d8e             Z6e G d9 d:e6             Z7 ed;       G d< d=e6             Z8 ed>       G d? d@e6             Z9 edA       G dB dCee6             Z:g dDZ;y)Ez!PyTorch Swinv2 Transformer model.    N)	dataclass)Tensornn   )initialization)ACT2FN)BackboneMixinfilter_output_hidden_states)GradientCheckpointingLayer)BackboneOutput)PreTrainedModel)ModelOutputauto_docstringlogging	torch_int)can_return_tuple   )Swinv2ConfigzP
    Swinv2 encoder's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   dZ
e	ej                  df   dz  ed<   dZe	ej                  df   dz  ed<   y)Swinv2EncoderOutputa  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   tupler   r        u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/swinv2/modeling_swinv2.pyr   r   (   s}     37u((4/6:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   r   zX
    Swinv2 model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	Swinv2ModelOutputa  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr   pooler_output.r   r   r   )r   r   r   r   r   r    r!   r"   r)   r   r#   r   r   r$   r%   r&   r(   r(   ?   s    	 37u((4/6.2M5$$t+2:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   r(   z,
    Swinv2 masked image model outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	Swinv2MaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlossreconstruction.r   r   r   )r   r   r   r   r,   r    r!   r"   r-   r   r#   r   r   r$   r%   r&   r+   r+   Y   s     &*D%

d
")/3NE%%,3:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   r+   z2
    Swinv2 outputs for image classification.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	Swinv2ImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr,   logits.r   r   r   )r   r   r   r   r,   r    r!   r"   r0   r   r#   r   r   r$   r%   r&   r/   r/   u   s     &*D%

d
")'+FE$+:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   r/   c                     | j                   \  }}}}| j                  |||z  |||z  ||      } | j                  dd      j                         j                  d|||      }|S )z2
    Partitions the given input into windows.
       r   shapeview	transpose
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowss          r&   window_partitionr@      st     /<.A.A+J|!&&Fk);8Lk[gM %%a+668==b+{\hiGNr%   c                     | j                   d   }| j                  d||z  ||z  |||      } | j                  dd      j                         j                  d|||      } | S )z?
    Merges windows to produce higher resolution features.
    r3   r2   r   r4   )r?   r:   r<   r=   r>   s        r&   window_reverserB      sg     ==$Lll2v4e{6JKYdfrsG1%00277FE<XGNr%   c            
            e Zd ZdZd fd	Zdej                  dededej                  fdZ	 	 dd	ej                  dz  d
ej                  dz  dedeej                     fdZ xZS )Swinv2EmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    c                 ~   t         |           t        |      | _        | j                  j                  }| j                  j
                  | _        |r4t        j                  t        j                  dd|j                              nd | _        |j                  r=t        j                  t        j                  d|dz   |j                              | _        nd | _        t        j                  |j                        | _        t        j"                  |j$                        | _        |j(                  | _        || _        y )Nr   )super__init__Swinv2PatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr    zeros	embed_dim
mask_tokenuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)selfrY   use_mask_tokenrJ   	__class__s       r&   rG   zSwinv2Embeddings.__init__   s     5f =++77//99O]",,u{{1a9I9I'JKcg))')||EKK;QR?TZTdTd4e'fD$'+D$LL!1!12	zz&"<"<= ++r%   
embeddingsr<   r=   returnc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr3         ?r   r   r2   bicubicF)sizemodealign_cornersdim)r5   rR   r    jit
is_tracingrX   r   reshapepermuter   
functionalinterpolater6   cat)rZ   r]   r<   r=   rJ   num_positionsclass_pos_embedpatch_pos_embedrf   
new_height	new_widthsqrt_num_positionss               r&   interpolate_pos_encodingz)Swinv2Embeddings.interpolate_pos_encoding   s`    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr%   Npixel_valuesbool_masked_posrt   c                    |j                   \  }}}}| j                  |      \  }}	| j                  |      }|j                         \  }
}}|K| j                  j                  |
|d      }|j                  d      j                  |      }|d|z
  z  ||z  z   }| j                  (|r|| j                  |||      z   }n|| j                  z   }| j                  |      }||	fS )Nr3         ?)r5   rI   rT   rb   rP   expand	unsqueezetype_asrR   rt   rW   )rZ   ru   rv   rt   _r>   r<   r=   r]   output_dimensionsr;   seq_lenmask_tokensmasks                 r&   forwardzSwinv2Embeddings.forward   s     *6););&<(,(=(=l(K%
%YYz*
!+!2
GQ&//00WbIK",,R088ED#sTz2[45GGJ##/''$*G*G
TZ\a*bb
'$*B*BB
\\*-
,,,r%   FNF)r   r   r   r   rG   r    r   intrt   r!   
BoolTensorboolr#   r   __classcell__r\   s   @r&   rD   rD      s    &&D5<< &D &DUX &D]b]i]i &DV 48).	-''$.- ))D0- #'	-
 
u||	-r%   rD   c                   v     e Zd ZdZ fdZd Zdej                  dz  deej                  ee
   f   fdZ xZS )rH   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _
        |d   |d   z  |d   |d   z  f| _        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)rF   rG   
image_sizerX   r>   rO   
isinstancecollectionsabcIterablerJ   rK   r   Conv2d
projection)rZ   rY   r   rX   r>   hidden_sizerJ   r\   s          r&   rG   zSwinv2PatchEmbeddings.__init__  s    !'!2!2F4E4EJ
$*$7$79I9Ik#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$&$Q-:a=8*Q-:VW=:XY))L+:^hir%   c                 n   || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|S )z9Pad pixel_values to be divisible by patch_size if needed.r   r   )rX   r   rk   pad)rZ   ru   r<   r=   
pad_valuess        r&   	maybe_padzSwinv2PatchEmbeddings.maybe_pad  s    4??1%%*T__Q/%$//!:L2LLMJ==,,\:FLDOOA&&!+Q4??1#5QRAS8S#STJ==,,\:FLr%   ru   Nr^   c                     |j                   \  }}}}| j                  |||      }| j                  |      }|j                   \  }}}}||f}|j                  d      j	                  dd      }||fS )Nr2   r   )r5   r   r   flattenr7   )rZ   ru   r|   r>   r<   r=   r]   r}   s           r&   r   zSwinv2PatchEmbeddings.forward%  s}    )5););&<~~lFEB__\2
(..1fe#UO''*44Q:
,,,r%   )r   r   r   r   rG   r   r    r!   r#   r   r   r   r   r   s   @r&   rH   rH     sF    j	-E$5$5$< 	-u||UZ[^U_G_A` 	-r%   rH   c            	            e Zd ZdZej
                  fdee   dedej                  ddf fdZ	d Z
d	ej                  d
eeef   dej                  fdZ xZS )Swinv2PatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutionrf   
norm_layerr^   Nc                     t         |           || _        || _        t	        j
                  d|z  d|z  d      | _         |d|z        | _        y )N   r2   Fbias)rF   rG   r   rf   r   Linear	reductionrT   )rZ   r   rf   r   r\   s       r&   rG   zSwinv2PatchMerging.__init__>  sI     01s7AG%@q3w'	r%   c                     |dz  dk(  xs |dz  dk(  }|r.ddd|dz  d|dz  f}t         j                  j                  ||      }|S )Nr2   r   r   )r   rk   r   )rZ   r9   r<   r=   
should_padr   s         r&   r   zSwinv2PatchMerging.maybe_padE  sU    qjAo:519>
Q519a!<JMM--mZHMr%   r9   input_dimensionsc                    |\  }}|j                   \  }}}|j                  ||||      }| j                  |||      }|d d dd ddd dd d f   }|d d dd ddd dd d f   }	|d d dd ddd dd d f   }
|d d dd ddd dd d f   }t        j                  ||	|
|gd      }|j                  |dd|z        }| j                  |      }| j                  |      }|S )Nr   r2   r   r3   r   )r5   r6   r   r    rm   r   rT   )rZ   r9   r   r<   r=   r;   rf   r>   input_feature_0input_feature_1input_feature_2input_feature_3s               r&   r   zSwinv2PatchMerging.forwardM  s   ((5(;(;%
C%**:vulS}feD'14a4Aq(89'14a4Aq(89'14a4Aq(89'14a4Aq(89		?O_Ve"fhjk%**:r1|;KL}5		-0r%   )r   r   r   r   r   rS   r#   r   ModulerG   r   r    r   r   r   r   s   @r&   r   r   1  sr    
 XZWcWc (s (# (299 (hl (U\\ U3PS8_ Y^YeYe r%   r   c            
            e Zd Zddgf fd	Z	 	 d
dej
                  dej                  dz  dedz  deej
                     fdZ	d	 Z
 xZS )Swinv2SelfAttentionr   c           
         t         |           ||z  dk7  rt        d| d| d      || _        t	        ||z        | _        | j                  | j
                  z  | _        t        |t        j                  j                        r|n||f| _        || _        t        j                  t        j                   dt        j"                  |ddf      z              | _        t        j&                  t        j(                  ddd	
      t        j*                  d	      t        j(                  d|d
            | _        | j/                         \  }}| j1                  d|d       | j1                  d|d       t        j(                  | j                  | j                  |j2                  
      | _        t        j(                  | j                  | j                  d
      | _        t        j(                  | j                  | j                  |j2                  
      | _        t        j:                  |j<                        | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()
   r   r2   i   Tr   )inplaceFrelative_coords_table)
persistentrelative_position_index) rF   rG   
ValueErrornum_attention_headsr   attention_head_sizeall_head_sizer   r   r   r   r:   pretrained_window_sizer   rM   r    logoneslogit_scale
Sequentialr   ReLUcontinuous_position_bias_mlpcreate_coords_table_and_indexregister_bufferqkv_biasquerykeyvaluerU   attention_probs_dropout_probrW   )	rZ   rY   rf   	num_headsr:   r   r   r   r\   s	           r&   rG   zSwinv2SelfAttention.__init__h  s   ?a#C5(^_h^iijk  $- #&sY#7 !558P8PP%k;??3K3KLKS^`kRl 	 '=#<<		"uzz9aQRBS7T2T(UV,.MMIIa4("''$*?3PY`eAf-
) :>9[9[9]6646KX]^68O\abYYt1143E3EFOO\
99T//1C1C%PYYt1143E3EFOO\
zz&"E"EFr%   Nr   attention_maskoutput_attentionsr^   c                 R   |j                   \  }}}| j                  |      j                  |d| j                  | j                        j                  dd      }| j                  |      j                  |d| j                  | j                        j                  dd      }| j                  |      j                  |d| j                  | j                        j                  dd      }	t        j                  j                  |d      t        j                  j                  |d      j                  dd      z  }
t        j                  | j                  t        j                  d            j!                         }|
|z  }
| j#                  | j$                        j                  d| j                        }|| j&                  j                  d         j                  | j(                  d   | j(                  d   z  | j(                  d   | j(                  d   z  d      }|j+                  ddd      j-                         }d	t        j.                  |      z  }|
|j1                  d      z   }
||j                   d   }|
j                  ||z  || j                  ||      |j1                  d      j1                  d      z   }
|
|j1                  d      j1                  d      z   }
|
j                  d| j                  ||      }
t        j                  j3                  |
d      }| j5                  |      }t        j6                  ||	      }|j+                  dddd
      j-                         }|j9                         d d | j:                  fz   }|j                  |      }|r||f}|S |f}|S )Nr3   r   r2   re   g      Y@)maxr      r   )r5   r   r6   r   r   r7   r   r   r   rk   	normalizer    clampr   mathr   expr   r   r   r:   rj   r8   sigmoidrz   softmaxrW   matmulrb   r   )rZ   r   r   r   r;   rf   r>   query_layer	key_layervalue_layerattention_scoresr   relative_position_bias_tablerelative_position_bias
mask_shapeattention_probscontext_layernew_context_layer_shapeoutputss                      r&   r   zSwinv2SelfAttention.forward  sx    )6(;(;%
CJJ}%T*b$":":D<T<TUYq!_ 	 HH]#T*b$":":D<T<TUYq!_ 	 JJ}%T*b$":":D<T<TUYq!_ 	 ==22;B2G"--JaJa2 Kb K

)B
 kk$"2"28LMQQS+k9'+'H'HIcIc'd'i'i(((
$ ">d>Z>Z>_>_`b>c!d!i!iQ$"2"21"55t7G7G7JTM]M]^_M`7`bd"
 "8!?!?1a!H!S!S!U!#emm4J&K!K+.D.N.Nq.QQ%'--a0J/44j(*d6N6NPSUX ((+55a8 9  0.2J2J12M2W2WXY2ZZ/44R9Q9QSVX[\ --//0@b/I ,,7 _kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r%   c                 8   t        j                  | j                  d   dz
   | j                  d   t         j                        j	                         }t        j                  | j                  d   dz
   | j                  d   t         j                        j	                         }t        j
                  t        j                  ||gd            j                  ddd      j                         j                  d      }| j                  d   dkD  rO|d d d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d d d dfxx   | j                  d   dz
  z  cc<   n`| j                  d   dkD  rN|d d d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d d d dfxx   | j                  d   dz
  z  cc<   |dz  }t        j                  |      t        j                  t        j                  |      dz         z  t        j                  d      z  }|j                  t!        | j"                  j%                               j&                        }t        j                  | j                  d         }t        j                  | j                  d         }t        j
                  t        j                  ||gd            }t        j(                  |d      }|d d d d d f   |d d d d d f   z
  }|j                  ddd      j                         }|d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   d| j                  d   z  dz
  z  cc<   |j+                  d	      }	||	fS )
Nr   r   dtypeij)indexingr2      rx   r3   )r    aranger:   int64floatstackmeshgridrj   r8   rz   r   signlog2absr   tonextr   
parametersr   r   sum)
rZ   relative_coords_hrelative_coords_wr   coords_hcoords_wcoordscoords_flattenrelative_coordsr   s
             r&   r   z1Swinv2SelfAttention.create_coords_table_and_index  s?   !LL4+;+;A+>+B)CTEUEUVWEX`e`k`klrrt!LL4+;+;A+>+B)CTEUEUVWEX`e`k`klrrtKK(9;L'MX\]^WQ1Z\Yq\	 	 &&q)A-!!Q1*-1L1LQ1ORS1SS-!!Q1*-1L1LQ1ORS1SS-a 1$!!Q1*-1A1A!1Dq1HH-!!Q1*-1A1A!1Dq1HH-"JJ,-

599EZ;[^a;a0bbeienenopeqq 	 !6 8 8d>_>_>j>j>l9m9s9s t << 0 0 34<< 0 0 34U^^Xx,@4PQvq1(At4~aqj7QQ)11!Q:EEG1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? "1"5"5b"9$&===r%   r   )r   r   r   rG   r    r   r!   r   r#   r   r   r   r   s   @r&   r   r   g  sm    TUWXSY G@ 48).	B||B ))D0B  $;	B
 
u||	BH#>r%   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )Swinv2SelfOutputc                     t         |           t        j                  ||      | _        t        j
                  |j                        | _        y N)rF   rG   r   r   denserU   r   rW   rZ   rY   rf   r\   s      r&   rG   zSwinv2SelfOutput.__init__  s6    YYsC(
zz&"E"EFr%   r   input_tensorr^   c                 J    | j                  |      }| j                  |      }|S r   r   rW   )rZ   r   r  s      r&   r   zSwinv2SelfOutput.forward  s$    

=1]3r%   r   r   r   rG   r    r   r   r   r   s   @r&   r   r     s2    G
U\\  RWR^R^ r%   r   c            
            e Zd Zd fd	Z	 	 d	dej
                  dej                  dz  dedz  deej
                     fdZ	 xZ
S )
Swinv2Attentionc           
          t         |           t        ||||t        |t        j
                  j                        r|n||f      | _        t        ||      | _	        y )NrY   rf   r   r:   r   )
rF   rG   r   r   r   r   r   rZ   r   output)rZ   rY   rf   r   r:   r   r\   s         r&   rG   zSwinv2Attention.__init__  sY    '#0+//2J2JK $:(*@A
	 'vs3r%   Nr   r   r   r^   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )rZ   r	  )rZ   r   r   r   self_outputsattention_outputr   s          r&   r   zSwinv2Attention.forward  sE     yy@QR;;|AF#%QR(88r%   r   r   )r   r   r   rG   r    r   r!   r   r#   r   r   r   s   @r&   r  r    sW    4  48).		||	 ))D0	  $;		
 
u||		r%   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Swinv2Intermediatec                    t         |           t        j                  |t	        |j
                  |z              | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )rF   rG   r   r   r   	mlp_ratior   r   
hidden_actstrr   intermediate_act_fnr   s      r&   rG   zSwinv2Intermediate.__init__  sa    YYsC(8(83(>$?@
f''-'-f.?.?'@D$'-'8'8D$r%   r   r^   c                 J    | j                  |      }| j                  |      }|S r   )r   r  rZ   r   s     r&   r   zSwinv2Intermediate.forward!  s&    

=100?r%   r  r   s   @r&   r  r    s#    9U\\ ell r%   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Swinv2Outputc                     t         |           t        j                  t	        |j
                  |z        |      | _        t        j                  |j                        | _	        y r   )
rF   rG   r   r   r   r  r   rU   rV   rW   r   s      r&   rG   zSwinv2Output.__init__)  sF    YYs6#3#3c#9:C@
zz&"<"<=r%   r   r^   c                 J    | j                  |      }| j                  |      }|S r   r  r  s     r&   r   zSwinv2Output.forward.  s$    

=1]3r%   r  r   s   @r&   r  r  (  s#    >
U\\ ell r%   r  c                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
Swinv2DropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probr^   Nc                 0    t         |           || _        y r   )rF   rG   r  )rZ   r  r\   s     r&   rG   zSwinv2DropPath.__init__<  s    "r%   r   c                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )N        r   r   )r   )r   device)
r  trainingr5   ndimr    randr   r!  floordiv)rZ   r   	keep_probr5   random_tensors        r&   r   zSwinv2DropPath.forward@  s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r%   c                      d| j                    S )Nzp=)r  rZ   s    r&   
extra_reprzSwinv2DropPath.extra_reprI  s    DNN#$$r%   )r   )r   r   r   r   r   rG   r    r   r   r  r+  r   r   s   @r&   r  r  5  sB    #% #$ #<U\\ <ell <%C %r%   r  c                        e Zd Z	 d fd	Zdeeeef   eeef   f   fdZd Zd Z	 dde	j                  deeef   ded	z  dee	j                  e	j                  f   fd
Z xZS )Swinv2Layerc           
      n   t         	|           || _        | j                  |j                  |j                  f||f      \  }}|d   | _        |d   | _        t        |||| j                  t        |t        j                  j                        r|n||f      | _        t        j                  ||j                        | _        |dkD  rt!        |      nt        j"                         | _        t'        ||      | _        t+        ||      | _        t        j                  ||j                        | _        y )Nr   r  epsr   )rF   rG   r   _compute_window_shiftr:   
shift_sizer  r   r   r   r   	attentionr   rS   layer_norm_epslayernorm_beforer  Identity	drop_pathr  intermediater  r	  layernorm_after)
rZ   rY   rf   r   r   drop_path_rater2  r   r:   r\   s
            r&   rG   zSwinv2Layer.__init__N  s    	 0"&"<"<!3!34z:6N#
Z 'q>$Q-(((0+//2J2JK $:(*@A
 !#Sf6K6K L;IC;O7UWU`U`Ub.vs;"63/!||CV5J5JKr%   r^   c                     t        | j                  |      D cg c]  \  }}t        ||       }}}t        | j                  ||      D cg c]  \  }}}||k  rdn| }}}}||fS c c}}w c c}}}w Nr   )zipr   min)rZ   target_window_sizetarget_shift_sizerwr:   sr2  s           r&   r1  z!Swinv2Layer._compute_window_shiftg  sy    -01F1FHZ-[\TQs1ay\\8;D<Q<QS^`q8rssWQ116aq(s
sJ&& ]ss   A*A0c                    | j                   dk  ryt        j                  |      }t        j                  |      }||| j                  z
  k\  j	                         ||| j                   z
  k\  j	                         z   }||| j                  z
  k\  j	                         ||| j                   z
  k\  j	                         z   }|dddddf   dz  |dddddf   z   j                  |      }t        || j                        }	|	j                  d| j                  | j                  z        }	|	j                  d      |	j                  d      z
  }
|
j                  |
dk7  d      j                  |
dk(  d      }
|
S )	u  Build the cyclic-shift attention mask for shifted-window MSA; returns None when shift_size is 0.

        Each (h, w) position belongs to one of 9 cyclic-shift regions (3 along each axis), encoded
        as ``h_region * 3 + w_region``. Regions per axis:
        - 0: indices ``[0, axis - window_size)``
        - 1: indices ``[axis - window_size, axis - shift_size)``
        - 2: indices ``[axis - shift_size, axis)``
        Implementation note: a single arithmetic pass on `torch.arange` (two comparisons +
        broadcast add) replaces the original 9-iteration nested-Python-loop slice-assignment —
        fully vectorised, no per-cell host-side scatter, no GPU↔host sync.
        r   Nr   r3   r   r2   g      Yr   )
r2  r    r   r:   longr   r@   r6   rz   masked_fill)rZ   r<   r=   r   h_idxw_idxh_regionw_regionimg_maskmask_windows	attn_masks              r&   get_attn_maskzSwinv2Layer.get_attn_maskl  se    ??aV$U#Vd&6&666<<>%6TXTcTcKcBcAiAiAkkUT%5%555;;=%RVRaRaJaAa@g@g@iiT1dD01A5tQPTAT8UUYYZ_`'$2B2BC#((T-=-=@P@P-PQ **1-0F0Fq0II	)))q.&AMMi[\n^ab	r%   c                     | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||      }||fS r<  )r:   r   rk   r   )rZ   r   r<   r=   	pad_right
pad_bottomr   s          r&   r   zSwinv2Layer.maybe_pad  s    %%0@0@(@@DDTDTT	&&$2B2B)BBdFVFVV
Ay!Z8
))-Dj((r%   r   r   r   Nc                    |\  }}|j                         \  }}}|}	|j                  ||||      }| j                  |||      \  }}
|j                  \  }}}}| j                  dkD  r1t        j                  || j                   | j                   fd      }n|}t        || j                        }|j                  d| j                  | j                  z  |      }| j                  |||j                        }||j                  |j                        }| j                  |||      }|d   }|j                  d| j                  | j                  |      }t        || j                  ||      }| j                  dkD  r/t        j                  || j                  | j                  fd      }n|}|
d   dkD  xs |
d   dkD  }|r|d d d |d |d d f   j                         }|j                  |||z  |      }| j!                  |      }|	| j#                  |      z   }| j%                  |      }| j'                  |      }|| j#                  | j)                  |            z   }|r	||d	   f}|S |f}|S )
Nr   )r   r2   )shiftsdimsr3   r   )r   r      r   )rb   r6   r   r5   r2  r    rollr@   r:   rN  r   r   r!  r3  rB   r8   r5  r7  r8  r	  r9  )rZ   r   r   r   r<   r=   r;   r|   channelsshortcutr   
height_pad	width_padshifted_hidden_stateshidden_states_windowsrM  attention_outputsr  attention_windowsshifted_windows
was_paddedlayer_outputlayer_outputss                          r&   r   zSwinv2Layer.forward  s    )"/"4"4"6
Ax  &**:vuhO$(NN=&%$P!z&3&9&9#:y!??Q$)JJ}tFVY]YhYhXhEipv$w!$1! !11FHXHX Y 5 : :2t?O?ORVRbRb?bdl m&&z9MDWDW&X	 !%:%A%ABI NN+@)_pNq,Q/,11"d6F6FHXHXZbc():D<L<LjZcd ??Q %

?DOOUYUdUdCelr s /]Q&;*Q-!*;
 1!WfWfufa2G H S S U-22:v~xX--.?@ 4>>-#@@((7{{<0$t~~d6J6J<6X'YY@Q'8';< YeWfr%   )r   r   r   r   )r   r   r   rG   r#   r   r1  rN  r   r    r   r   r   r   r   s   @r&   r-  r-  M  s    qrL2'eTYZ]_bZbTcejknpsksetTtNu '
2) */	5||5  S/5  $;	5
 
u||U\\)	*5r%   r-  c            
       |     e Zd Z	 d fd	Z	 d	dej
                  deeef   dedz  deej
                     fdZ	 xZ
S )
Swinv2Stagec	           
      |   t         |           || _        || _        g }	t	        |      D ]?  }
t        ||||||
   |
dz  dk(  rdn|j                  dz  |      }|	j                  |       A t        j                  |	      | _
        |& |||t        j                        | _        d| _        y d | _        d| _        y )Nr2   r   )rY   rf   r   r   r:  r2  r   )rf   r   F)rF   rG   rY   rf   ranger-  r:   appendr   
ModuleListblocksrS   
downsamplepointing)rZ   rY   rf   r   depthr   r7  rj  r   ri  iblockr\   s               r&   rG   zSwinv2Stage.__init__  s     	u 
	!A!1#(|!"Q!1&2D2D2I'=E MM% 
	! mmF+ !()9sr||\DO  #DOr%   r   r   r   Nr^   c                     |\  }}t        | j                        D ]  \  }} ||||      }|d   } |}	| j                  )|dz   dz  |dz   dz  }}
|||
|f}| j                  |	|      }n||||f}||	|f}|r|dd  z  }|S )Nr   r   r2   )	enumerateri  rj  )rZ   r   r   r   r<   r=   rm  layer_modulerb  !hidden_states_before_downsamplingheight_downsampledwidth_downsampledr}   stage_outputss                 r&   r   zSwinv2Stage.forward  s     )(5 	-OA|( !M *!,M	- -:)??&5;aZA4EPQ	VWGW 1!'0BDU V OO,MO_`M!' >&(IK\]]12..Mr%   r  r   )r   r   r   rG   r    r   r#   r   r   r   r   r   s   @r&   rd  rd    sU    mn@ */	||  S/  $;	
 
u||	r%   rd  c                        e Zd Zd fd	Z	 	 	 	 ddej
                  deeef   dedz  dedz  dedz  dedz  d	ee	z  fd
Z
 xZS )Swinv2Encoderc                 >   t         	|           t        |j                        | _        || _        | j
                  j                  |j                  }t        j                  d|j                  t        |j                        d      D cg c]  }|j                          }}g }t        | j                        D ]  }t        |t        |j                  d|z  z        |d   d|z  z  |d   d|z  z  f|j                  |   |j                   |   |t        |j                  d |       t        |j                  d |dz           || j                  dz
  k  rt"        nd ||         }|j%                  |        t'        j(                  |      | _        d| _        y c c}w )Nr   cpu)r!  r2   r   )rY   rf   r   rl  r   r7  rj  r   F)rF   rG   lendepths
num_layersrY   pretrained_window_sizesr    linspacer:  r   itemrf  rd  r   rO   r   r   rg  r   rh  layersgradient_checkpointing)
rZ   rY   rK   r}  xdprr  i_layerstager\   s
            r&   rG   zSwinv2Encoder.__init__  sw   fmm,;;..:&,&D&D#!&63H3H#fmmJ\ej!klAqvvxllT__- 	!G((1g:56"+A,1g:">	!QRT[Q[@\!]mmG, **73c&--"9:S}QX[\Q\A]=^_29DOOa<O2O-VZ'>w'G	E MM% 	! mmF+&+## ms   Fr   r   r   Noutput_hidden_states(output_hidden_states_before_downsamplingreturn_dictr^   c                    |rdnd }|rdnd }|rdnd }	|rE|j                   \  }
}} |j                  |
g|| }|j                  dddd      }||fz  }||fz  }t        | j                        D ]  \  }} ||||      }|d   }|d   }|d   }|d   |d   f}|rP|rN|j                   \  }
}} |j                  |
g|d   |d   f| }|j                  dddd      }||fz  }||fz  }nI|rG|sE|j                   \  }
}} |j                  |
g|| }|j                  dddd      }||fz  }||fz  }|s|	|dd  z  }	 |st        d |||	|fD              S t        |||	|	      S )
Nr$   r   r   r   r2   r   r3   c              3   $   K   | ]  }|| 
 y wr   r$   ).0vs     r&   	<genexpr>z(Swinv2Encoder.forward.<locals>.<genexpr>R  s      = s   )r   r   r   r   )r5   r6   rj   rp  r  r#   r   )rZ   r   r   r   r  r  r  all_hidden_statesall_reshaped_hidden_statesall_self_attentionsr;   r|   r   reshaped_hidden_staterm  rq  rb  rr  r}   s                      r&   r   zSwinv2Encoder.forward  sB    #7BD+?RT"$5b4)6)<)<&J;$6M$6$6z$bDT$bVa$b!$9$A$A!Q1$M!-!11&+@*BB&(5  	9OA|( !M *!,M0=a0@- -a 0 1" 57H7LM#(P-N-T-T*
A{ )O(I(N(N)"3A"68I!8L!M)OZ)% )>(E(EaAq(Q%!&G%II!*/D.FF*%.V-:-@-@*
A{(:(:(::(fHX(fZe(f%(=(E(EaAq(Q%!m%55!*/D.FF* #}QR'88#A 	9D  '):<OQkl   #++*#=	
 	
r%   ))r   r   r   r   )FFFT)r   r   r   rG   r    r   r#   r   r   r   r   r   r   s   @r&   rw  rw     s    ,: */,1@E#'C
||C
  S/C
  $;	C

 #TkC
 37+C
 D[C
 
$	$C
r%   rw  c                   h     e Zd ZU eed<   dZdZdZdZdgZ	 e
j                          fd       Z xZS )Swinv2PreTrainedModelrY   swinv2ru   )imageTrd  c                 "   t         |   |       t        |t              rX|j                  t        j                  |j                         |j                   t        j                  |j                         yyt        |t              rt        j                  |j                  t        j                  d             |j                         \  }}t        j                  |j                  |       t        j                  |j                   |       yy)zInitialize the weightsNr   )rF   _init_weightsr   rD   rP   initzeros_rR   r   	constant_r   r   r   r   copy_r   r   )rZ   moduler   r   r\   s       r&   r  z#Swinv2PreTrainedModel._init_weightsi  s     	f%f./  ,F--.))5F667 6 34NN6--txx|<=C=a=a=c:!#:JJv335JKJJv557NO	 5r%   )r   r   r   r   r"   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modulesr    no_gradr  r   r   s   @r&   r  r  `  sD     $O!&*#&U]]_P Pr%   r  c                        e Zd Zd fd	Zd Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  de	d	e	dz  d
e
ez  fd       Z xZS )Swinv2Modelc                    t         |   |       || _        t        |j                        | _        t        |j                  d| j
                  dz
  z  z        | _        t        ||      | _
        t        || j                  j                        | _        t        j                  | j                  |j                         | _        |rt        j$                  d      nd| _        | j)                          y)a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        r2   r   )r[   r/  N)rF   rG   rY   rz  r{  r|  r   rO   num_featuresrD   r]   rw  rL   encoderr   rS   r4  	layernormAdaptiveAvgPool1dpooler	post_init)rZ   rY   add_pooling_layerr[   r\   s       r&   rG   zSwinv2Model.__init__|  s     	 fmm, 0 0119L3M MN*6.Q$VT__-G-GHd&7&7V=R=RS1Bb**1- 	r%   c                 .    | j                   j                  S r   r]   rI   r*  s    r&   get_input_embeddingsz Swinv2Model.get_input_embeddings      ///r%   Nru   rv   r   r  rt   r  r^   c                 @   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  |||      \  }}	| j                  ||	|||      }
|
d   }| j                  |      }d}| j                  7| j                  |j                  dd            }t        j                  |d      }|s||f|
dd z   }|S t        |||
j                  |
j                  |
j                        S )	z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rv   rt   )r   r  r  r   r   r2   )r   r)   r   r   r   )rY   r   r  r  r   r]   r  r  r  r7   r    r   r(   r   r   r   )rZ   ru   rv   r   r  rt   r  kwargsembedding_outputr   encoder_outputssequence_outputpooled_outputr	  s                 r&   r   zSwinv2Model.forward  sJ    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY?@@-1__/Tl .= .
** ,,/!5# ' 
 *!,..9;;" KK(A(A!Q(GHM!MM-;M%}58KKFM -')77&11#2#I#I
 	
r%   )TFNNNNFN)r   r   r   rG   r  r   r    r!   r   r   r#   r(   r   r   r   s   @r&   r  r  y  s    *0  2637)-,0).#'6
''$.6
 ))D06
  $;	6

 #Tk6
 #'6
 D[6
 
"	"6
 6
r%   r  a~  
        Swinv2 Model with a decoder on top for masked image modeling, as proposed in
    [SimMIM](https://huggingface.co/papers/2111.09886).

        <Tip>

        Note that we provide a script to pre-train this model on custom data in our [examples
        directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

        </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
       Z xZS )Swinv2ForMaskedImageModelingc                    t         |   |       t        |dd      | _        t	        |j
                  d|j                  dz
  z  z        }t        j                  t        j                  ||j                  dz  |j                  z  d      t        j                  |j                              | _        | j                          y )NFT)r  r[   r2   r   )in_channelsout_channelsr   )rF   rG   r  r  r   rO   r|  r   r   r   encoder_strider>   PixelShuffledecoderr  )rZ   rY   r  r\   s      r&   rG   z%Swinv2ForMaskedImageModeling.__init__  s     !&ERVW6++aF4E4E4I.JJK}}II(v7L7La7ORXReRe7est OOF112	
 	r%   Nru   rv   r   r  rt   r  r^   c                    ||n| j                   j                  }| j                  ||||||      }|d   }	|	j                  dd      }	|	j                  \  }
}}t        j                  |dz        x}}|	j                  |
|||      }	| j                  |	      }d}|| j                   j                  | j                   j                  z  }|j                  d||      }|j                  | j                   j                  d      j                  | j                   j                  d      j                  d      j                         }t        j                  j!                  ||d	      }||z  j#                         |j#                         d
z   z  | j                   j$                  z  }|s|f|dd z   }||f|z   S |S t'        |||j(                  |j*                  |j,                        S )a  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, Swinv2ForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = Swinv2ForMaskedImageModeling.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 256, 256]
        ```N)rv   r   r  rt   r  r   r   r2   r`   r3   none)r   gh㈵>)r,   r-   r   r   r   )rY   r  r  r7   r5   r   r%  ri   r  r   rX   repeat_interleaverz   r8   r   rk   l1_lossr   r>   r+   r   r   r   )rZ   ru   rv   r   r  rt   r  r  r   r  r;   r>   sequence_lengthr<   r=   reconstructed_pixel_valuesmasked_im_lossrb   r   reconstruction_lossr	  s                        r&   r   z$Swinv2ForMaskedImageModeling.forward  s   P &1%<k$++BYBY+++/!5%=#  
 "!*)33Aq94C4I4I1
L/OS$899)11*lFTYZ &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN02WQR[@F3A3M^%.YSYY.5!//))#*#A#A
 	
r%   r  )r   r   r   rG   r   r    r!   r   r   r#   r+   r   r   r   s   @r&   r  r    s       2637)-,0).#'S
''$.S
 ))D0S
  $;	S

 #TkS
 #'S
 D[S
 
0	0S
 S
r%   r  a  
    Swinv2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune SwinV2 on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                        e Zd Z fdZe	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  dededz  d	e	e
z  fd
       Z xZS )Swinv2ForImageClassificationc                 >   t         |   |       |j                  | _        t        |      | _        |j                  dkD  r4t        j                  | j                  j                  |j                        nt        j                         | _	        | j                          y r<  )rF   rG   
num_labelsr  r  r   r   r  r6  
classifierr  )rZ   rY   r\   s     r&   rG   z%Swinv2ForImageClassification.__init__T  sx      ++!&) GMFWFWZ[F[BIIdkk..0A0ABacalalan 	
 	r%   Nru   labelsr   r  rt   r  r^   c                 V   ||n| j                   j                  }| j                  |||||      }|d   }	| j                  |	      }
d}|| j	                  ||
| j                         }|s|
f|dd z   }||f|z   S |S t        ||
|j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r  rt   r  r   r2   )r,   r0   r   r   r   )	rY   r  r  r  loss_functionr/   r   r   r   )rZ   ru   r  r   r  rt   r  r  r   r  r0   r,   r	  s                r&   r   z$Swinv2ForImageClassification.forwardb  s    " &1%<k$++BYBY++/!5%=#  
  
/%%ffdkkBDY,F)-)9TGf$EvE*!//))#*#A#A
 	
r%   r  )r   r   r   rG   r   r    r!   
LongTensorr   r#   r/   r   r   r   s   @r&   r  r  D  s       26*.)-,0).#',
''$.,
   4',
  $;	,

 #Tk,
 #',
 D[,
 
,	,,
 ,
r%   r  zO
    Swinv2 backbone, to be used with frameworks like DETR and MaskFormer.
    c                   x     e Zd Z fdZd Zeee	 	 	 d
dede	dz  de	dz  de	dz  de
f
d	                     Z xZS )Swinv2Backbonec           	      h   t         |   |       |j                  gt        t	        |j
                              D cg c]  }t        |j                  d|z  z         c}z   | _        t        |      | _	        t        || j                  j                        | _        | j                          y c c}w )Nr2   )rF   rG   rO   rf  rz  r{  r   r  rD   r]   rw  rL   r  r  )rZ   rY   rm  r\   s      r&   rG   zSwinv2Backbone.__init__  s     #--.X]^abhbobo^pXq1rST#f6F6FA6M2N1rr*62$VT__-G-GH 	 2ss   "B/c                 .    | j                   j                  S r   r  r*  s    r&   r  z#Swinv2Backbone.get_input_embeddings  r  r%   Nru   r   r  r  r^   c                 
   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |      \  }}| j                  |||dd|      }|r|j                  n|d   }	d}
t        | j                  |	      D ]  \  }}|| j                  v s|
|fz  }
 |s|
f}|r	||d   fz  }|r	||d   fz  }|S t        |
|r|j                  nd|j                        S )	a  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swinv2-tiny-patch4-window8-256", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 2048, 7, 7]
        ```NT)r   r  r  r  r3   r$   r   r2   )feature_mapsr   r   )rY   r  r  r   r]   r  r   r=  stage_namesout_featuresr   r   r   )rZ   ru   r   r  r  r  r  r   r   r   r  r  hidden_stater	  s                 r&   r   zSwinv2Backbone.forward  sA   J &1%<k$++BYBY$8$D $++JjJj 	 2C1N-TXT_T_TqTq-1__\-J**,,/!%59#  
 ;F667SU;#&t'7'7#G 	0E<)))/	0 "_F#71:-' 71:-'M%3G'//T))
 	
r%   )NNN)r   r   r   rG   r  r   r
   r   r   r   r   r   r   r   s   @r&   r  r    s    0   *.,0#'F
F
  $;F
 #Tk	F

 D[F
 
F
  ! F
r%   r  )r  r  r  r  r  )<r   collections.abcr   r   dataclassesr   r    r   r    r   r  activationsr   backbone_utilsr	   r
   modeling_layersr   modeling_outputsr   modeling_utilsr   utilsr   r   r   r   utils.genericr   configuration_swinv2r   
get_loggerr   loggerr   r(   r+   r/   r@   rB   r   rD   rH   r   r   r   r  r  r  r  r-  rd  rw  r  r  r  r  r  __all__r$   r%   r&   <module>r     s   (   !   & ! H 9 . - D D - . 
		H	% 
 H+ H H  
 H H H& 
 Hk H H* 
 H+ H H,	Y-ryy Y-z'-BII '-T3 3lE>")) E>R
ryy 
bii 6  	299 	%RYY %0t")) tn9, 9x]
BII ]
@ PO P P0 P
' P
 P
f 
e
#8 e
e
P <
#8 <
<
~ 
W
]$9 W

W
tr%   