
    ^j                        d dl Zd dlZd dl mZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z# ddl$m%Z%  G d dejL                        Z' ed      e G d de                    Z( ed      e G d de                    Z) ed      e G d de                    Z* ed      e G d d e                    Z+ G d! d"ejL                        Z, G d# d$ejL                        Z- G d% d&ejL                        Z. G d' d(ejL                        Z/	 	 dLd)ejL                  d*ej`                  d+ej`                  d,ej`                  d-ej`                  dz  d.e1dz  d/e1d0ee   fd1Z2 G d2 d3ejL                        Z3 G d4 d5ejL                        Z4d6 Z5d7 Z6 G d8 d9e      Z7 G d: d;e      Z8e G d< d=e             Z9 G d> d?e9      Z:e G d@ dAe9             Z; edB       G dC dDe9             Z< edE       G dF dGe9             Z= edH       G dI dJee9             Z>g dKZ?y)M    N)Callable)	dataclass)nn   )initialization)ACT2FN)BackboneMixinfilter_output_hidden_states)GradientCheckpointingLayer)BackboneOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstring	torch_int)can_return_tuplemerge_with_config_defaults)OutputRecordercapture_outputs   )
SwinConfigc                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
SwinDropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probreturnNc                 0    t         |           || _        y N)super__init__r   )selfr   	__class__s     q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/swin/modeling_swin.pyr!   zSwinDropPath.__init__1   s    "    hidden_statesc                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )N        r   r   )r   dtypedevice)
r   trainingshapendimtorchrandr*   r+   floordiv)r"   r&   	keep_probr-   random_tensors        r$   forwardzSwinDropPath.forward5   s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r%   c                      d| j                    S )Nzp=)r   )r"   s    r$   
extra_reprzSwinDropPath.extra_repr>   s    DNN#$$r%   )r(   )__name__
__module____qualname____doc__floatr!   r/   Tensorr5   strr7   __classcell__r#   s   @r$   r   r   *   sB    #% #$ #<U\\ <ell <%C %r%   r   zN
    Swin encoder's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   dZ
e	ej                  df   dz  ed<   dZe	ej                  df   dz  ed<   y)SwinEncoderOutputa  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlast_hidden_state.r&   
attentionsreshaped_hidden_states)r8   r9   r:   r;   rD   r/   FloatTensor__annotations__r&   tuplerE   rF    r%   r$   rC   rC   B   s}     37u((4/6:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   rC   zV
    Swin model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinModelOutputa  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NrD   pooler_output.r&   rE   rF   )r8   r9   r:   r;   rD   r/   rG   rH   rM   r&   rI   rE   rF   rJ   r%   r$   rL   rL   X   s    	 37u((4/6.2M5$$t+2:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   rL   z*
    Swin masked image model outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinMaskedImageModelingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
        Masked image modeling (MLM) loss.
    reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Reconstructed pixel values.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlossreconstruction.r&   rE   rF   )r8   r9   r:   r;   rP   r/   rG   rH   rQ   r&   rI   rE   rF   rJ   r%   r$   rO   rO   q   s     &*D%

d
")/3NE%%,3:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   rO   z0
    Swin outputs for image classification.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	SwinImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    NrP   logits.r&   rE   rF   )r8   r9   r:   r;   rP   r/   rG   rH   rT   r&   rI   rE   rF   rJ   r%   r$   rS   rS      s     &*D%

d
")'+FE$+:>M5**C/047>7;Je'',-4;CGE%"3"3S"89D@Gr%   rS   c            
            e Zd ZdZd fd	Zdej                  dededej                  fdZ	 	 dd	ej                  dz  d
ej                  dz  dedeej                     fdZ xZS )SwinEmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    c                 l   t         |           t        |      | _        | j                  j                  }| j                  j
                  | _        |r4t        j                  t        j                  dd|j                              nd | _        |j                  r4t        j                  t        j                  d||j                              nd | _        t        j                  |j                        | _        t        j"                  |j$                        | _        |j(                  | _        || _        y )Nr   )r    r!   SwinPatchEmbeddingspatch_embeddingsnum_patches	grid_size
patch_gridr   	Parameterr/   zeros	embed_dim
mask_tokenuse_absolute_embeddingsposition_embeddings	LayerNormnormDropouthidden_dropout_probdropout
patch_sizeconfig)r"   ri   use_mask_tokenrZ   r#   s       r$   r!   zSwinEmbeddings.__init__   s     3F ;++77//99O]",,u{{1a9I9I'JKcg LRKiKiBLLQV5E5EFGos 	  LL!1!12	zz&"<"<= ++r%   
embeddingsheightwidthr   c                 (   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S |j                   d   }|| j
                  z  }|| j
                  z  }t        |dz        }	| j                  j                  d|	|	|      }
|
j                  dddd      }
t        j                  j                  |
||fdd	      }
|
j                  dddd      j                  dd|      S )
z
        Interpolate pre-trained position encodings to support higher-resolution images at inference.
        Unlike ViT, Swin has no CLS token, so position embeddings cover patch positions only.
        r         ?r   r      bicubicF)sizemodealign_corners)r-   rb   r/   jit
is_tracingrh   r   reshapepermuter   
functionalinterpolateview)r"   rk   rl   rm   rZ   num_positionsdim
new_height	new_widthsqrt_num_positionspatch_pos_embeds              r$   interpolate_pos_encodingz'SwinEmbeddings.interpolate_pos_encoding   s   
 !&&q)0066q9 yy##%+*F6UZ?+++r"t.
T__,	&}c'9:22::1>PRdfij)11!Q1=--33i(	 4 
 &&q!Q2772sCCr%   Npixel_valuesbool_masked_posr   c                    |j                   \  }}}}| j                  |      \  }}	| j                  |      }|j                         \  }
}}|K| j                  j                  |
|d      }|j                  d      j                  |      }|d|z
  z  ||z  z   }| j                  (|r|| j                  |||      z   }n|| j                  z   }| j                  |      }||	fS )Nro   g      ?)r-   rY   rd   rs   r`   expand	unsqueezetype_asrb   r   rg   )r"   r   r   r   _num_channelsrl   rm   rk   output_dimensions
batch_sizeseq_lenmask_tokensmasks                 r$   r5   zSwinEmbeddings.forward   s     *6););&<(,(=(=l(K%
%YYz*
!+!2
GQ&//00WbIK",,R088ED#sTz2[45GGJ##/''$*G*G
TZ\a*bb
'$*B*BB
\\*-
,,,r%   F)NF)r8   r9   r:   r;   r!   r/   r=   intr   rG   
BoolTensorboolrI   r5   r?   r@   s   @r$   rV   rV      s    "D5<< D DUX D]b]i]i DB 48).	-''$.- ))D0- #'	-
 
u||	-r%   rV   c                   v     e Zd ZdZ fdZd Zdej                  dz  deej                  ee
   f   fdZ xZS )rX   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _
        |d   |d   z  |d   |d   z  f| _        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)r    r!   
image_sizerh   r   r_   
isinstancecollectionsabcIterablerZ   r[   r   Conv2d
projection)r"   ri   r   rh   r   hidden_sizerZ   r#   s          r$   r!   zSwinPatchEmbeddings.__init__   s    !'!2!2F4E4EJ
$*$7$79I9Ik#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$&$Q-:a=8*Q-:VW=:XY))L+:^hir%   c                 n   || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||      }|S )z9Pad pixel_values to be divisible by patch_size if needed.r   r   )rh   r   rz   pad)r"   r   rl   rm   
pad_valuess        r$   	maybe_padzSwinPatchEmbeddings.maybe_pad  s    4??1%%*T__Q/%$//!:L2LLMJ==,,\:FLDOOA&&!+Q4??1#5QRAS8S#STJ==,,\:FLr%   r   Nr   c                     |j                   \  }}}}| j                  |||      }| j                  |      }|j                   \  }}}}||f}|j                  d      j	                  dd      }||fS )Nrq   r   )r-   r   r   flatten	transpose)r"   r   r   r   rl   rm   rk   r   s           r$   r5   zSwinPatchEmbeddings.forward  s}    )5););&<~~lFEB__\2
(..1fe#UO''*44Q:
,,,r%   )r8   r9   r:   r;   r!   r   r/   rG   rI   r=   r   r5   r?   r@   s   @r$   rX   rX      sF    j	-E$5$5$< 	-u||UZ[^U_G_A` 	-r%   rX   c                        e Zd ZdZdeddf fdZdej                  dededej                  fd	Zdej                  d
e	eef   dej                  fdZ
 xZS )SwinPatchMergingzd
    Patch Merging Layer.

    Args:
        dim (`int`):
            Number of input channels.
    r~   r   Nc                     t         |           t        j                  d|z  d|z  d      | _        t        j
                  d|z        | _        y )N   rq   Fbias)r    r!   r   Linear	reductionrc   rd   )r"   r~   r#   s     r$   r!   zSwinPatchMerging.__init__*  s>    1s7AG%@LLS)	r%   input_featurerl   rm   c           
      ~    |dz  dk(  s|dz  dk(  r,t         j                  j                  |ddd|dz  d|dz  f      }|S )zPPad input feature map to be divisible by 2 in both spatial dimensions if needed.rq   r   r   )r   rz   r   )r"   r   rl   rm   s       r$   r   zSwinPatchMerging.maybe_pad/  sL    QJ!OaMM--maAuqyRSU[^_U_=`aMr%   input_dimensionsc                    |\  }}|j                   \  }}}|j                  ||||      }| j                  |||      }t        j                  t        d      D 	cg c]%  }t        d      D ]  }	|d d |	d d|d dd d f    ' c}	}d      }|j                  |dd|z        }| j                  |      }| j                  |      }|S c c}	}w )Nrq   ro   r~   r   )r-   r|   r   r/   catrangerd   r   )
r"   r   r   rl   rm   r   r~   r   colrows
             r$   r5   zSwinPatchMerging.forward5  s    ((5(;(;%
C%**:vulS}feD		<A!HYSPUVWPXY]1cf1fcf1fa/0Y0Y_a
 &**:r1|;KL		-0}5 Zs   *C
)r8   r9   r:   r;   r   r!   r/   r=   r   rI   r5   r?   r@   s   @r$   r   r   !  su    *C *D *
u|| S  QVQ]Q] U\\ U3PS8_ Y^YeYe r%   r   c                   v     e Zd ZdZdedeeef   f fdZdej                  fdZ	dej                  fdZ
 xZS )SwinRelativePositionBiasa  
    Relative position bias for Swin's window-based attention, following the style of BeitRelativePositionBias.

    Unlike BeiT, Swin has no CLS token, so the table covers exactly (2*ws_h-1)*(2*ws_w-1) unique
    relative positions. The lookup index is purely determined by window_size (static), so it is stored
    as a non-persistent buffer (recomputed from config on load, never serialised). The table values
    are learned parameters and must be re-read on every forward call.
    	num_headswindow_sizec                 8   t         |           || _        |d   |d   z  | _        t	        j
                  t        j                  d|d   z  dz
  d|d   z  dz
  z  |            | _        | j                  d| j                         j                  d      d       y )Nr   r   rq   relative_position_indexro   F)
persistent)r    r!   r   window_arear   r]   r/   r^   relative_position_bias_tableregister_buffer_create_relative_position_indexr|   )r"   r   r   r#   s      r$   r!   z!SwinRelativePositionBias.__init__S  s    &&q>KN:,.LLKK[^+a/AA4F4JKYW-
)
 	%00277; 	 	
r%   r   c                    t        j                  | j                  d         }t        j                  | j                  d         }t        j                  t        j                  ||gd            }t        j
                  |d      }|d d d d d f   |d d d d d f   z
  }|j                  ddd      j                         }|d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   d| j                  d   z  dz
  z  cc<   |j                  d      S )Nr   r   ij)indexingrq   ro   )	r/   aranger   stackmeshgridr   ry   
contiguoussum)r"   coords_hcoords_wcoordscoords_flattenrelative_coordss         r$   r   z8SwinRelativePositionBias._create_relative_position_indexb  s)   << 0 0 34<< 0 0 34U^^Xx,@4PQvq1(At4~aqj7QQ)11!Q:EEG 	1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? ""2&&r%   c                     | j                   | j                     }|j                  | j                  | j                  d      }|j	                  ddd      j                         j                  d      S )Nro   rq   r   r   )r   r   r|   r   ry   r   r   )r"   relative_position_biass     r$   r5   z SwinRelativePositionBias.forwards  sd    !%!B!B4C_C_!`!7!<!<T=M=MtO_O_ac!d%--aA6AACMMaPPr%   )r8   r9   r:   r;   r   rI   r!   r/   r=   r   r5   r?   r@   s   @r$   r   r   I  sE    
# 
E#s(O 
' '"Q Qr%   r   modulequerykeyvalueattention_maskscalingrg   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |dt        j                        j                  |j                        }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nro         rq   r   )r~   r*   )pr,   r   )rs   r/   matmulr   r   rz   softmaxfloat32tor*   rg   r,   r   )
r   r   r   r   r   r   rg   r   attn_weightsattn_outputs
             r$   eager_attention_forwardr   y  s     **R.D( <<s}}Q':;gEL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r%   c                        e Zd Zdedededef fdZ	 ddej                  dej                  dz  d	e	e
   d
eej                  ej                  f   fdZ xZS )SwinAttentionri   r   num_attention_headsr   c                    t         |           || _        || _        ||z  | _        |j
                  | _        | j                  dz  | _        d| _        t        j                  |||j                        | _        t        j                  |||j                        | _        t        j                  |||j                        | _        t        j                  ||      | _        t!        |||f      | _        y )Nr   Fr   )r    r!   ri   r   head_dimattention_probs_dropout_probattention_dropoutr   	is_causalr   r   qkv_biasq_projk_projv_projo_projr   r   )r"   ri   r   r   r   r#   s        r$   r!   zSwinAttention.__init__  s    #6 #'::!'!D!D}}d*ii[vOii[vOii[vOii[9&>?RU`bmTn&o#r%   Nr&   r   r   r   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                         }	|g|j                   d   }
|d   |
z  }|d   }|j                  d      j                  d      j                  |dddd      j                  dd||      }|	|z   }n|	}t        j                  | j                  j                  t              } || ||||f| j                   sdn| j"                  | j$                  d|\  }} |j                  g |d j'                         }| j)                  |      }||fS )Nro   r   rq   r   r(   )rg   r   )r-   r   r   r|   r   r   r   r   r   r   rx   r   get_interfaceri   _attn_implementationr   r,   r   r   r   r   )r"   r&   r   r   input_shapehidden_shapequery_states
key_statesvalue_statesr   num_windowsr   r   combined_maskattention_interfacer   r   s                    r$   r5   zSwinAttention.forward  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST "&!<!<!>%(..q1K$Q;6J!!nG ((+1
BB3Q1	  3^CM2M(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r%   r   )r8   r9   r:   r   r   r!   r/   r=   rG   r   r   rI   r5   r?   r@   s   @r$   r   r     s    pz p pRU pdg p& 482)||2) ))D02) +,	2)
 
u||U\\)	*2)r%   r   c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )SwinMLPri   r~   c                    t         |           t        |j                     | _        t        j                  |t        |j                  |z              | _	        t        j                  t        |j                  |z        |      | _
        y r   )r    r!   r   
hidden_actactivation_fnr   r   r   	mlp_ratiofc1fc2)r"   ri   r~   r#   s      r$   r!   zSwinMLP.__init__  se    #F$5$5699S#f&6&6&<"=>99S!1!1C!78#>r%   r&   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r  r   r  )r"   r&   s     r$   r5   zSwinMLP.forward  s4    /**=9/r%   )
r8   r9   r:   r   r   r!   r/   r=   r5   r?   r@   s   @r$   r   r     s1    ?z ? ?U\\ ell r%   r   c                     | j                   \  }}}}| j                  |||z  |||z  ||      } | j                  dd      j                         j                  d|||      }|S )z2
    Partitions the given input into windows.
    rq   r   ro   r-   r|   r   r   )r   r   r   rl   rm   r   windowss          r$   window_partitionr    st     /<.A.A+J|!&&Fk);8Lk[gM %%a+668==b+{\hiGNr%   c                     | j                   d   }| j                  d||z  ||z  |||      } | j                  dd      j                         j                  d|||      } | S )z?
    Merges windows to produce higher resolution features.
    ro   rq   r   r  )r  r   rl   rm   r   s        r$   window_reverser
    sg     ==$Lll2v4e{6JKYdfrsG1%00277FE<XGNr%   c                       e Zd Z	 	 ddededeeef   dededef fdZ	 ddej                  d	eeef   d
e
dee   dej                  f
dZdeeef   ddfdZdededej                  dej                   dej                  dz  f
dZdej                  dededeej                  eedf   f   fdZddej                  de
dej                  fdZ xZS )	SwinLayerri   r~   input_resolutionr   drop_path_rate
shift_sizec                    t         |           t        ||||j                        | _        t        j                  ||j                        | _        t        j                  ||j                        | _	        t        ||      | _        t        j                  |j                        | _        |j                  | _        || _        || _        |dkD  rt#        |      | _        y t        j$                         | _        y )N)r   epsr(   )r    r!   r   r   	attentionr   rc   layer_norm_epslayernorm_beforelayernorm_afterr   mlpre   rf   rg   r  r  r   Identity	drop_path)r"   ri   r~   r  r   r  r  r#   s          r$   r!   zSwinLayer.__init__  s     	&vsI6K]K]^ "Sf6K6K L!||CV5J5JK63'zz&"<"<=!--$ 09G#9Mn5SUS^S^S`r%   r&   r   always_partitionr   r   c                    |s| j                  |       |\  }}|j                         \  }}}	|}
| j                  |      }|j                  ||||	      }| j	                  |||      \  }}|j
                  \  }}}}t        | j                  |      | j                        }|j                  d| j                  | j                  z  |	      }| j                  |||j                  |j                        } | j                  ||fi |\  }}| j                  |      }|j                  d| j                  | j                  |	      }| j                  t        || j                  ||      d      }|d   dkD  s|d   dkD  r|d d d |d |d d f   j                         }|j                  |||z  |	      }|
| j!                  |      z   }|}| j#                  |      }| j%                  |      }| j                  |      |z   }||fS )Nro   r)   T)reverser   r      )set_shift_and_window_sizers   r  r|   r   r-   r  cyclic_shiftr   get_attn_maskr*   r+   r  rg   r
  r   r  r  r  )r"   r&   r   r  r   rl   rm   r   r   channelsshortcutr   
height_pad	width_padhidden_states_windows	attn_maskattention_outputr   attention_windowsresiduals                       r$   r5   zSwinLayer.forward  s-     **+;<("/"4"4"6
Ax --m<%**:vuhO %)NN=&%$P!z&3&9&9#:y! 01B1B=1QSWScSc d 5 : :2t?O?ORVRbRb?bdl m&&	)<)<EZEaEa ' 
	 *88My)c\b)c&,<<(89,11"d6F6FHXHXZbc --,d.>.>
IV`d . 
 a=1
1 1 1!WfWfufa2G H S S U-22:v~xX 4>>2C#DD ,,];/]3h>l**r%   Nc                    t        |      | j                  k  rgt        d      | _        t        j
                  j                         r(t	        j                   t	        j                  |            n
t        |      | _        yy)zQClamp window and shift sizes when the window is larger than the input resolution.r   N)minr   r   r  r/   rv   rw   tensor)r"   r  s     r$   r  z#SwinLayer.set_shift_and_window_sizeD  s\     D$4$44'lDO=BYY=Q=Q=S		%,,'789Y\]mYn  5r%   rl   rm   r*   r+   c                    | j                   dk  ryt        j                  ||      }t        j                  ||      }||| j                  z
  k\  j	                         ||| j                   z
  k\  j	                         z   }||| j                  z
  k\  j	                         ||| j                   z
  k\  j	                         z   }|dddddf   dz  |dddddf   z   j                  |      }	t        |	| j                        }
|
j                  d| j                  | j                  z        }
|
j                  d      |
j                  d      z
  }|j                  |dk7  d      j                  |dk(  d	      }|S )
u  Build the cyclic-shift attention mask for shifted-window MSA; returns None when shift_size is 0.

        Each (h, w) position belongs to one of 9 cyclic-shift regions (3 along each axis), encoded
        as ``h_region * 3 + w_region``. Regions per axis:
        - 0: indices ``[0, axis - window_size)``
        - 1: indices ``[axis - window_size, axis - shift_size)``
        - 2: indices ``[axis - shift_size, axis)``
        Implementation note: a single arithmetic pass on `torch.arange` (two comparisons +
        broadcast add) replaces the original 9-iteration nested-Python-loop slice-assignment —
        fully vectorised, no per-cell host-side scatter, no GPU↔host sync.
        r   N)r+   r   ro   r   rq   g      Yr(   )
r  r/   r   r   longr   r  r|   r   masked_fill)r"   rl   rm   r*   r+   h_idxw_idxh_regionw_regionimg_maskmask_windowsr&  s               r$   r   zSwinLayer.get_attn_maskL  si    ??aVF3U62Vd&6&666<<>%6TXTcTcKcBcAiAiAkkUT%5%555;;=%RVRaRaJaAa@g@g@iiT1dD01A5tQPTAT8UUYYZ_`'$2B2BC#((T-=-=@P@P-PQ **1-0F0Fq0II	)))q.&AMMi[\n^ab	r%   .c                     | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||      }||fS )zHPad feature map so both spatial dimensions are divisible by window_size.r   )r   r   rz   r   )r"   r&   rl   rm   	pad_right
pad_bottomr   s          r$   r   zSwinLayer.maybe_pade  s    %%0@0@(@@DDTDTT	&&$2B2B)BBdFVFVV
Ay!Z8
))-Dj((r%   r  c                     | j                   dkD  r:|rdnd}t        j                  ||| j                   z  || j                   z  fd      }|S )zOApply a cyclic shift along the spatial dimensions for shifted-window attention.r   r   ro   )r   rq   )shiftsdims)r  r/   roll)r"   r&   r  	directions       r$   r  zSwinLayer.cyclic_shiftm  sM    ??Q$"I!JJ!DOO3Y5PQM
 r%   )r(   r   r   )r8   r9   r:   r   r   rI   r<   r!   r/   r=   r   r   r   r5   r  r*   r+   r   r   r  r?   r@   s   @r$   r  r     sn    !$aa a  S/	a
 a a a0 "'	-+||-+  S/-+ 	-+
 +,-+ 
-+^%S/ d C  EKK QVQ]Q] bgbnbnqubu 2)u|| )S ) )QVW\WcWcejknpsksetWtQu )	%,, 	 	RWR^R^ 	r%   r  c                   <    e Zd Zdededeeef   dededee   f fdZde	j                  d	e	j                  d
ededede	j                  fdZ	 	 dde	j                  deeef   dededee   dee	j                  e	j                  e	j                  dz  f   fdZ xZS )	SwinStageri   r~   r  depthr   r  c                    t         	|           || _        t        j                  t        |      D cg c]-  }t        ||||||   |dz  dk(  rdn|j                  dz        / c}      | _        | ||      | _	        y d | _	        y c c}w )Nrq   r   )ri   r~   r  r   r  r  r   )
r    r!   ri   r   
ModuleListr   r  r   blocks
downsample)
r"   ri   r~   r  r@  r   r  rD  ir#   s
            r$   r!   zSwinStage.__init__z  s     	mm u
  !%5'#,Q<%&UaZqf6H6HA6M

 2<1G*-T
s   2B
r&   !hidden_states_before_downsamplingrl   rm   (output_hidden_states_before_downsamplingr   c                     |r|||}}}n%| j                   ||dz   dz  |dz   dz  }}}n|||}}}|j                  \  }	}
}|j                  |	|||      j                  dddd      j	                         S )u  
        Select the spatial hidden states for this stage and reshape from (B, L, C) to (B, C, H, W).

        The chosen state and its resolution depend on output_hidden_states_before_downsampling:
        - True  → pre-downsampling states at (height, width) — used by the backbone.
        - False → post-downsampling states at half the resolution (if a downsampler exists).
        r   rq   r   r   )rD  r-   r|   ry   r   )r"   r&   rF  rl   rm   rG  spatial_statehwr   r   r   s               r$   get_reshaped_hidden_statesz$SwinStage.get_reshaped_hidden_states  s     4"CVUa1M__("/&1*1BUQYSTDTa1M"/a1M%2%8%8"
A{!!*aK@HHAqRST__aar%   r   r  r   Nc                     |\  }}d }| j                   D ]  }	 |	||fd|i|\  }} |}
| j                  | j                  |
|      }| j                  ||
|||      }|||fS )Nr  )rC  rD  rL  )r"   r&   r   r  rG  r   rl   rm   last_attn_weightslayer_modulerF  rF   s               r$   r5   zSwinStage.forward  s     )  KK 	L/;/0BR0V\0,M,	
 -:)??& OO,MO_`M!%!@!@<feMu"
 46GGGr%   )FF)r8   r9   r:   r   r   rI   listr<   r!   r/   r=   r   rL  r   r   r5   r?   r@   s   @r$   r?  r?  y  s$   RR R  S/	R
 R R ;R8b||b ,1<<b 	b
 b 37b 
b: "'9>H||H  S/H 	H
 37H +,H 
u||U\\5<<$+>>	?Hr%   r?  c                        e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZdZ eedd	       eed
d	      dZdZddgZ ej*                          fd       Z xZS )SwinPreTrainedModelri   swinr   )imageTr?  Fr   )indexcapture_initial_hidden_staterq   )r&   rE   rY   z(attention\.self\.relative_position_indexz:attention\.relative_position_bias\.relative_position_indexc                    t         |   |       t        |t              rX|j                  t        j                  |j                         |j                   t        j                  |j                         yyt        |t              r]t        j                  |j                         t        j                  |j                  |j                         j                  d             yy)zInitialize the weightsNro   )r    _init_weightsr   rV   r`   initzeros_rb   r   r   copy_r   r   r|   )r"   r   r#   s     r$   rX  z!SwinPreTrainedModel._init_weights  s     	f%fn-  ,F--.))5F667 6 89KK;;<JJv55v7]7]7_7d7deg7hi :r%   )r8   r9   r:   r   rH   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_can_compile_fullgraphr   r?  _can_record_outputs_input_embed_layer"_keys_to_ignore_on_load_unexpectedr/   no_gradrX  r?   r@   s   @r$   rR  rR    s    $O!&*#$N "&! (	Y]^
 %YaV[\	 , 	4E*&
 U]]_
j 
jr%   rR  c                        e Zd Zdedeeef   f fdZe ed      e		 	 	 dde
j                  deeef   ded	ed
edee   defd                     Z xZS )SwinEncoderri   r[   c                    t         |   |       t        |j                        | _        || _        t        t        |j                              D cg c]4  }|j                  |z  t        t        |j                        dz
  d      z  6 }}t        j                  t        | j                        D cg c]  }t        |t        |j                  d|z  z        |d   d|z  z  |d   d|z  z  f|j                  |   |j                  |   |t        |j                  d |       t        |j                  d |dz           || j                  dz
  k  rt         nd        c}      | _        | j%                          y c c}w c c}w )Nr   rq   r   )ri   r~   r  r@  r   r  rD  )r    r!   lendepths
num_layersri   r   r   r  maxr   rB  r?  r   r_   r   r   layers	post_init)r"   ri   r[   rE  dpr	layer_idxr#   s         r$   r!   zSwinEncoder.__init__  sd    fmm,SXY\]c]j]jYkSlmav$$q(3s6==/AA/Eq+IImmmm "'t!7  !F,,q)|;<&/lq)|&DiPQlWXZcWcFd%e --	2$..y9!#fmmJY&?"@3v}}UdW`cdWdGeCfg4=RS@S4S/Z^
 	 ns   9E:3B*E?F)tie_last_hidden_statesr&   r   r  output_hidden_statesrG  r   r   c                 N   d}|rH|j                   \  }}	}
 |j                  |g||
 j                  dddd      j                         }|f}| j                  D ]>  } |||f||d|\  }}}	|r||fz  }|j
                  )|d   dz   dz  |d   dz   dz  f}@ t        ||      S )a  
        input_dimensions (`tuple[int, int]`):
            Spatial `(height, width)` of the patch grid entering the encoder.
        always_partition (`bool`, *optional*, defaults to `False`):
            If `True`, always apply window partitioning regardless of input resolution.
        output_hidden_states_before_downsampling (`bool`, *optional*, defaults to `False`):
            If `True`, `reshaped_hidden_states` contains pre-downsampling feature maps.
        Nr   r   r   rq   r  rG  )rD   rF   )r-   r|   ry   r   rq  rD  rC   )r"   r&   r   r  rv  rG  r   all_reshaped_hidden_statesr   r   r   stem_spatialrO  reshaped_hidden_states                 r$   r5   zSwinEncoder.forward  s   ( &*" *7)<)<&J;""":N0@N+NVVWXZ[]^`abmmo  +7& KK 	dL6B 7 "29a	7
 73M0! $*/D.FF*&&2%5a%81%<$BEUVWEX[\E\abDb#c 	d !+#=
 	
r%   )FFF)r8   r9   r:   r   rI   r   r!   r   r   r   r/   r=   r   r   r   rC   r5   r?   r@   s   @r$   rk  rk    s    z eCHo *  E2
 "'%*9>+
||+
  S/+
 	+

 #+
 37+
 +,+
 
+
  3  +
r%   rk  c                        e Zd Zd	 fd	Zee	 	 	 d
dej                  dz  dej                  dz  de	de
e   def
d              Z xZS )	SwinModelc                    t         |   |       || _        t        |j                        | _        t        |j                  d| j
                  dz
  z  z        | _        t        ||      | _
        t        || j                  j                        | _        t        j                  | j                  |j                         | _        |rt        j$                  d      nd| _        | j)                          y)a  
        add_pooling_layer (`bool`, *optional*, defaults to `True`):
            Whether or not to apply pooling layer.
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether or not to create and apply mask tokens in the embedding layer.
        rq   r   )rj   r  N)r    r!   ri   rm  rn  ro  r   r_   num_featuresrV   rk   rk  r\   encoderr   rc   r  	layernormAdaptiveAvgPool1dpoolerrr  )r"   ri   add_pooling_layerrj   r#   s       r$   r!   zSwinModel.__init__>  s     	 fmm, 0 0119L3M MN(O"64??+E+EFd&7&7V=R=RS1Bb**1- 	r%   Nr   r   r   r   r   c                    |j                  d| j                  j                        }| j                  |||      \  }} | j                  ||fd|i|}|j
                  }	| j                  |	      }	d}
| j                  7| j                  |	j                  dd            }
t        j                  |
d      }
t        |	|
|j                  |j                  |j                        S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        rv  r   r   Nr   rq   )rD   rM   r&   rE   rF   )popri   rv  rk   r  rD   r  r  r   r/   r   rL   r&   rE   rF   )r"   r   r   r   r   rv  embedding_outputr   encoder_outputssequence_outputpooled_outputs              r$   r5   zSwinModel.forwardS  s      &zz*@$++BbBbc-1__/Tl .= .
** '$,,
 "6
 	
 *;;..9;;" KK(A(A!Q(GHM!MM-;M-')77&11#2#I#I
 	
r%   )TFNNF)r8   r9   r:   r!   r   r   r/   rG   r   r   r   r   rL   r5   r?   r@   s   @r$   r}  r}  <  s{    *  2637).	(
''$.(
 ))D0(
 #'	(

 +,(
 
(
  (
r%   r}  ad  
    Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                        e Zd Z fdZee	 	 	 d	dej                  dz  dej                  dz  de	de
e   def
d              Z xZS )
SwinForMaskedImageModelingc                    t         |   |       t        |dd      | _        t	        |j
                  d|j                  dz
  z  z        }t        j                  t        j                  ||j                  dz  |j                  z  d      t        j                  |j                              | _        | j                          y )NFT)r  rj   rq   r   )in_channelsout_channelsr   )r    r!   r}  rS  r   r_   ro  r   
Sequentialr   encoder_strider   PixelShuffledecoderrr  )r"   ri   r  r#   s      r$   r!   z#SwinForMaskedImageModeling.__init__  s     fdS	6++aF4E4E4I.JJK}}II(v7L7La7ORXReRe7est OOF112	
 	r%   Nr   r   r   r   r   c                 |    | j                   |f||d|}|j                  }|j                  dd      }|j                  \  }}}	t	        j
                  |	dz        x}
}|j                  |||
|      }| j                  |      }d}|| j                  j                  | j                  j                  z  }|j                  d||      }|j                  | j                  j                  d      j                  | j                  j                  d      j                  d      j                         }t        j                  j!                  ||d      }||z  j#                         |j#                         d	z   z  | j                  j$                  z  }t'        |||j(                  |j*                  |j,                  
      S )a  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, SwinForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swin-base-simmim-window6-192")
        >>> model = SwinForMaskedImageModeling.from_pretrained("microsoft/swin-base-simmim-window6-192")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```r  r   rq   rp   Nro   none)r   gh㈵>)rP   rQ   r&   rE   rF   )rS  rD   r   r-   mathr1   rx   r  ri   r   rh   repeat_interleaver   r   r   rz   l1_lossr   r   rO   r&   rE   rF   )r"   r   r   r   r   outputsr  r   r   sequence_lengthrl   rm   reconstructed_pixel_valuesmasked_im_lossrs   r   reconstruction_losss                    r$   r5   z"SwinForMaskedImageModeling.forward  s   L $))
+%=
 	
 "33)33Aq94C4I4I1
L/OS$899)11*lFTYZ &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN,5!//))#*#A#A
 	
r%   r  )r8   r9   r:   r!   r   r   r/   rG   r   r   r   r   rO   r5   r?   r@   s   @r$   r  r    s       2637).	H
''$.H
 ))D0H
 #'	H

 +,H
 
'H
  H
r%   r  a  
    Swin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune Swin on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                        e Zd Z fdZee	 	 	 d	dej                  dz  dej                  dz  de	de
e   def
d              Z xZS )
SwinForImageClassificationc                 >   t         |   |       |j                  | _        t        |      | _        |j                  dkD  r4t        j                  | j                  j                  |j                        nt        j                         | _	        | j                          y )Nr   )r    r!   
num_labelsr}  rS  r   r   r  r  
classifierrr  )r"   ri   r#   s     r$   r!   z#SwinForImageClassification.__init__  sx      ++f%	 EKDUDUXYDYBIIdii,,f.?.?@_a_j_j_l 	
 	r%   Nr   labelsr   r   r   c                     | j                   |fd|i|}|j                  }| j                  |      }d}| | j                  ||| j                  fi |}t        |||j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r   N)rP   rT   r&   rE   rF   )	rS  rM   r  loss_functionri   rS   r&   rE   rF   )	r"   r   r  r   r   r  r  rT   rP   s	            r$   r5   z"SwinForImageClassification.forward  s     $))
%=
 
  --/%4%%ffdkkLVLD(!//))#*#A#A
 	
r%   r  )r8   r9   r:   r!   r   r   r/   rG   
LongTensorr   r   r   rS   r5   r?   r@   s   @r$   r  r    s{      26*.).	!
''$.!
   4'!
 #'	!

 +,!
 
#!
  !
r%   r  zM
    Swin backbone, to be used with frameworks like DETR and MaskFormer.
    c            	       v     e Zd ZdgZdef fdZeeede	j                  dee   defd                     Z xZS )SwinBackbonezswin.layernorm.*ri   c           	         t         |   |       |j                  gt        t	        |j
                              D cg c]  }t        |j                  d|z  z         c}z   | _        t        |d      | _	        i }t        | j                  | j                        D ]  \  }}t        j                  |      ||<    t        j                  |      | _        | j#                          y c c}w )Nrq   F)r  )r    r!   r_   r   rm  rn  r   r  r}  rS  zipout_featuresr!  r   rc   
ModuleDicthidden_states_normsrr  )r"   ri   rE  r  stager   r#   s         r$   r!   zSwinBackbone.__init__5  s     #--.X]^abhbobo^pXq1rST#f6F6FA6M2N1rrf>	 !#&t'8'8$--#H 	DE<)+l)C&	D#%==1D#E  	 2ss   "C(r   r   r   c                     | j                   |fddd|}d}t        | j                  |j                        D ]  \  }}|| j                  v s|j
                  \  }}}	}
|j                  dddd      j                         }|j                  ||	|
z  |      } | j                  |   |      }|j                  ||	|
|      }|j                  dddd      j                         }||fz  } t        ||j                  |j                        S )	a  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> processor = AutoImageProcessor.from_pretrained("shi-labs/nat-mini-in1k-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swin-tiny-patch4-window7-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 7, 7]
        ```
        Trx  rJ   r   rq   r   r   )feature_mapsr&   rE   )rS  r  stage_namesrF   r  r-   ry   r   r|   r  r   rE   )r"   r   r   r  r  r  hidden_stater   r   rl   rm   s              r$   r5   zSwinBackbone.forwardD  s0   H $))
!59
 	
 #&t'7'79W9W#X 	0E<))):F:L:L7
L&%+33Aq!Q?JJL+00Ve^\Z>t77>|L+00VULY+33Aq!Q?JJL/	0 %!88))
 	
r%   )r8   r9   r:   _keys_to_ignore_on_load_missingr   r!   r   r
   r   r/   r=   r   r   r   r5   r?   r@   s   @r$   r  r  -  sd     (;&;#z   7
ll7
 +,7
 
	7
  ! 7
r%   r  )r  r  r}  rR  r  )Nr(   )@collections.abcr   r  r   dataclassesr   r/   r    r   rY  activationsr   backbone_utilsr	   r
   modeling_layersr   modeling_outputsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   r   configuration_swinr   Moduler   rC   rL   rO   rS   rV   rX   r   r   r=   r<   r   r   r   r  r
  r  r?  rR  rk  r}  r  r  r  __all__rJ   r%   r$   <module>r     s  *   $ !   & ! H 9 . F & O O I E *%299 %0 
 H H H  
 Hk H H& 
 HK H H* 
 H H H*M-RYY M-`'-")) '-T%ryy %P-Qryy -Ql !%II%<<% 
% <<	%
 LL4'% T\% % '(%8C)BII C)Lbii 	v* vrMH* MH` (j/ (j (jVD
% D
N @
# @
 @
F 	[
!4 [
[
| 2
!4 2
2
j 
L
="5 L

L
^r%   