
    ^j                      B   d Z ddlZddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,  e#jZ                  e.      Z/ e"d      e G d de                     Z0 e"d      e G d de                     Z1e"e G d de                     Z2dejf                  dejf                  fd Z4d!ejf                  dejf                  fd"Z5d#e,d$e6fd%Z7d^d&e6e8z  d'e9fd(Z: G d) d*e	jv                        Z< G d+ d,e	jz                        Z> G d- d.e	jv                        Z? G d/ d0e	jv                        Z@ G d1 d2e	jv                        ZA G d3 d4e	jv                        ZB G d5 d6e	jv                        ZC G d7 d8e	jv                        ZD G d9 d:e	jv                        ZE	 d_d;e	jv                  d<ejf                  d=ejf                  d>ejf                  d?ejf                  dz  d@eFdAeFfdBZG G dC dDe	jv                        ZH G dE dFe	jv                        ZI G dG dHe	jv                        ZJ G dI dJe	jv                        ZK G dK dLe	jv                        ZL G dM dNe      ZM G dO dPe	jv                        ZN G dQ dRe	jv                        ZOe" G dS dTe             ZP e"dU       G dV dWeP             ZQ e"dX       G dY dZeP             ZRe" G d[ d\eP             ZSg d]ZTy)`zPyTorch ALIGN model.    N)Callable)	dataclass)Any)nn   )initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithNoAttentionBaseModelOutputWithPooling(BaseModelOutputWithPoolingAndNoAttention)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )AlignConfigAlignTextConfigAlignVisionConfigz}
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   y)AlignVisionModelOutputz
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
        The image embeddings obtained by applying the projection layer to the pooler_output.
    Nimage_embedslast_hidden_statehidden_states)__name__
__module____qualname____doc__r"   torchFloatTensor__annotations__r#   r$   tuple     s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/align/modeling_align.pyr!   r!   .   sN    
 .2L%##d*126u((4/659M5**+d29r.   r!   ze
    Base class for text model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)AlignTextModelOutputz
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
        The text embeddings obtained by applying the projection layer to the pooler_output.
    Ntext_embedsr#   r$   
attentions)r%   r&   r'   r(   r2   r)   r*   r+   r#   r$   r,   r3   r-   r.   r/   r1   r1   ?   sh    
 -1K""T)026u((4/659M5**+d2926Je''(4/6r.   r1   c                      e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   dZeed<   dZeed	<   d
ee   fdZy)AlignOutputar  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`AlignTextModel`].
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The output of [`AlignVisionModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`AlignTextModel`].
    vision_model_output (`BaseModelOutputWithPoolingAndNoAttention`):
        The output of the [`AlignVisionModel`].
    Nlosslogits_per_imagelogits_per_textr2   r"   text_model_outputvision_model_outputreturnc                 H     t         fd j                         D              S )Nc              3   d   K   | ]'  }|d vr|   nt        |      j                          ) yw))r9   r:   N)getattrto_tuple).0kselfs     r/   	<genexpr>z'AlignOutput.to_tuple.<locals>.<genexpr>p   s=      
  LLDGRYZ^`aRbRkRkRmm
s   -0)r,   keysrB   s   `r/   r?   zAlignOutput.to_tupleo   s#     
YY[
 
 	
r.   )r%   r&   r'   r(   r6   r)   r*   r+   r7   r8   r2   r"   r9   r   r:   r   r,   r   r?   r-   r.   r/   r5   r5   Q   s    & &*D%

d
")15e''$.504OU&&-4,0K""T)0-1L%##d*14818DHAH
%* 
r.   r5   logitsr;   c                     t         j                  j                  | t        j                  t        |       | j                        d      S )Ndeviceg?)label_smoothing)r   
functionalcross_entropyr)   arangelenrI   )rF   s    r/   contrastive_lossrO   x   s5    ==&&vu||CKPVP]P]/^ps&ttr.   
similarityc                 Z    t        |       }t        | j                               }||z   dz  S )Ng       @)rO   t)rP   caption_loss
image_losss      r/   
align_lossrU   |   s,    #J/L!*,,.1J:%,,r.   confignum_channelsc                     | j                   }|| j                  z  }t        |t        ||dz  z         |z  |z        }|d|z  k  r||z  }t        |      S )z<
    Round number of filters based on depth multiplier.
       g?)depth_divisorwidth_coefficientmaxint)rV   rW   divisornew_dims       r/   round_filtersr`      sf     ""GF,,,L'3|gk9:gEOPG |##7w<r.   kernel_sizeadjustc                     t        | t              r| | f} | d   dz  | d   dz  f}|r|d   dz
  |d   |d   dz
  |d   fS |d   |d   |d   |d   fS )aJ  
    Utility function to get the tuple padding value for the depthwise convolution.

    Args:
        kernel_size (`int` or `tuple`):
            Kernel size of the convolution layers.
        adjust (`bool`, *optional*, defaults to `True`):
            Adjusts padding value to apply to right and bottom sides of the input.
    r   rY   r   )
isinstancer]   )ra   rb   corrects      r/   correct_padrf      s}     +s#"K01~"KNa$78G
Q
GAJNGAJGG
GAJ
GAJ??r.   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )AlignVisionEmbeddingszL
    A module that corresponds to the stem module of the original work.
    rV   c                    t         |           t        |d      | _        t	        j
                  d      | _        t	        j                  |j                  | j                  dddd      | _	        t	        j                  | j                  |j                  |j                  	      | _        t        |j                     | _        y )
N    )r   r   r   r   paddingr   rY   validFra   striderl   bias)epsmomentum)super__init__r`   out_dimr   	ZeroPad2drl   Conv2drW   convolutionBatchNorm2dbatch_norm_epsbatch_norm_momentum	batchnormr	   
hidden_act
activationrB   rV   	__class__s     r/   rt   zAlignVisionEmbeddings.__init__   s    $VR0||L9991QPW^c
 &:O:OZ`ZtZtu !2!23r.   pixel_valuesr;   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S N)rl   rx   r|   r~   )rB   r   featuress      r/   forwardzAlignVisionEmbeddings.forward   sA    <<-##H->>(+??8,r.   )
r%   r&   r'   r(   r   rt   r)   Tensorr   __classcell__r   s   @r/   rh   rh      s0    	40 	4ELL U\\ r.   rh   c                   .     e Zd Z	 	 	 	 	 	 	 d fd	Z xZS )AlignVisionDepthwiseConv2dc	                 @    ||z  }	t         
|   ||	|||||||	       y )N)	in_channelsout_channelsra   ro   rl   dilationgroupsrp   padding_mode)rs   rt   )rB   r   depth_multiplierra   ro   rl   r   rp   r   r   r   s             r/   rt   z#AlignVisionDepthwiseConv2d.__init__   s=     #%55#%#% 	 
	
r.   )r   r   r   r   r   Tzeros)r%   r&   r'   rt   r   r   s   @r/   r   r      s$     
 
r.   r   c                   l     e Zd ZdZdedededef fdZdej                  dej                  fd	Z
 xZS )
AlignVisionExpansionLayerz_
    This corresponds to the expansion phase of each block in the original implementation.
    rV   in_dimru   ro   c                     t         |           t        j                  ||ddd      | _        t        j
                  ||j                        | _        t        |j                     | _
        y )Nr   sameFr   r   ra   rl   rp   )num_featuresrq   )rs   rt   r   rw   expand_convry   rz   	expand_bnr	   r}   
expand_act)rB   rV   r   ru   ro   r   s        r/   rt   z"AlignVisionExpansionLayer.__init__   sZ    99 
 W&BWBWX !2!23r.   r$   r;   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   rB   r$   s     r/   r   z!AlignVisionExpansionLayer.forward   s4    ((7}56r.   )r%   r&   r'   r(   r   r]   rt   r)   r*   r   r   r   r   s   @r/   r   r      sH    
40 
4# 
4 
4UX 
4U%6%6 5<< r.   r   c            
       p     e Zd ZdZdededededef
 fdZdej                  d	ej                  fd
Z xZS )AlignVisionDepthwiseLayerzk
    This corresponds to the depthwise convolution phase of each block in the original implementation.
    rV   r   ro   ra   adjust_paddingc                 b   t         |           || _        | j                  dk(  rdnd}t        ||      }t	        j
                  |      | _        t        ||||d      | _        t	        j                  ||j                  |j                        | _        t        |j                     | _        y )	NrY   rm   r   )rb   rk   Frn   r   rq   rr   )rs   rt   ro   rf   r   rv   depthwise_conv_padr   depthwise_convry   rz   r{   depthwise_normr	   r}   depthwise_act)	rB   rV   r   ro   ra   r   conv_padrl   r   s	           r/   rt   z"AlignVisionDepthwiseLayer.__init__   s     	"kkQ.7Fk.A"$,,w"?8FHSX
 !nnV%:%:VE_E_
 $F$5$56r.   r$   r;   c                     | j                   dk(  r| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S )NrY   )ro   r   r   r   r   r   s     r/   r   z!AlignVisionDepthwiseLayer.forward  sT    ;;! 33MBM++M:++M:**=9r.   r%   r&   r'   r(   r   r]   boolrt   r)   r*   r   r   r   r   s   @r/   r   r      sZ    7!7 7 	7
 7 7,	U%6%6 	5<< 	r.   r   c            	       n     e Zd ZdZd
dedededef fdZdej                  dej                  fd	Z xZS )AlignVisionSqueezeExciteLayerzl
    This corresponds to the Squeeze and Excitement phase of each block in the original implementation.
    rV   r   
expand_dimexpandc                    t         |           |r|n|| _        t        dt	        ||j
                  z              | _        t        j                  d      | _	        t        j                  | j                  | j                  dd      | _        t        j                  | j                  | j                  dd      | _        t        |j                     | _        t        j                          | _        y )Nr   )output_sizer   )r   r   ra   rl   )rs   rt   dimr\   r]   squeeze_expansion_ratiodim_ser   AdaptiveAvgPool2dsqueezerw   reducer   r	   r}   
act_reduceSigmoid
act_expand)rB   rV   r   r   r   r   s        r/   rt   z&AlignVisionSqueezeExciteLayer.__init__%  s    !':V!S&*H*H!HIJ++:ii	
 ii	
 !!2!23**,r.   r$   r;   c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }t        j                  ||      }|S r   )r   r   r   r   r   r)   mul)rB   r$   inputss      r/   r   z%AlignVisionSqueezeExciteLayer.forward:  sc    ]3M26M26		&-8r.   )Fr   r   s   @r/   r   r      sH    '0 '# '3 'X\ '*
U%6%6 
5<< 
r.   r   c                        e Zd ZdZdedededededef fdZd	e	j                  d
e	j                  de	j                  fdZ xZS )AlignVisionFinalBlockLayerz[
    This corresponds to the final phase of each block in the original implementation.
    rV   r   ru   ro   	drop_rateid_skipc                    t         |           |dk(  xr | | _        t        j                  ||ddd      | _        t        j                  ||j                  |j                        | _	        t        j                  |      | _        y )Nr   r   Fr   r   )p)rs   rt   apply_dropoutr   rw   project_convry   rz   r{   
project_bnDropoutdropout)rB   rV   r   ru   ro   r   r   r   s          r/   rt   z#AlignVisionFinalBlockLayer.__init__L  sz     	#q[8[II 
 .. f&;&;fF`F`
 zzI.r.   
embeddingsr$   r;   c                     | j                  |      }| j                  |      }| j                  r| j                  |      }||z   }|S r   )r   r   r   r   )rB   r   r$   s      r/   r   z"AlignVisionFinalBlockLayer.forward]  sG    ))-86 LL7M)J6Mr.   r%   r&   r'   r(   r   r]   floatr   rt   r)   r*   r   r   r   r   s   @r/   r   r   G  sj    /'/14/?B/LO/\a/lp/"%"3"3 EDUDU Z_ZfZf r.   r   c                        e Zd ZdZdededededededed	ed
ef fdZde	j                  de	j                  fdZ xZS )AlignVisionBlocka  
    This corresponds to the block module of original the EfficientNet vision encoder implementation.

    Args:
        config ([`AlignVisionConfig`]):
            Model configuration class.
        in_dim (`int`):
            Number of input channels.
        out_dim (`int`):
            Number of output channels.
        stride (`int`):
            Stride size to be used in convolution layers.
        expand_ratio (`int`):
            Expand ratio to set the output dimensions for the expansion and squeeze-excite layers.
        kernel_size (`int`):
            Kernel size for the depthwise convolution layer.
        drop_rate (`float`):
            Dropout rate to be used in the final phase of each block.
        id_skip (`bool`):
            Whether to apply dropout and sum the final hidden states with the input embeddings during the final phase
            of each block. Set to `True` for the first block of each stage.
        adjust_padding (`bool`):
            Whether to apply padding to only right and bottom side of the input kernel before the depthwise convolution
            operation, set to `True` for inputs with odd input sizes.
    rV   r   ru   ro   expand_ratiora   r   r   r   c
                 l   t         |           || _        | j                  dk7  | _        ||z  }
| j                  rt	        |||
|      | _        t        || j                  r|
n||||	      | _        t        |||
| j                        | _	        t        || j                  r|
n|||||      | _        y )Nr   )rV   r   ru   ro   )rV   r   ro   ra   r   )rV   r   r   r   )rV   r   ru   ro   r   r   )rs   rt   r   r   r   	expansionr   r   r   squeeze_exciter   
projection)rB   rV   r   ru   ro   r   ra   r   r   r   expand_in_dimr   s              r/   rt   zAlignVisionBlock.__init__  s     	(''1,-;;6fmFDN 8$(KK=V#)
 <&]4;;
 5$(KK=V
r.   r$   r;   c                     |}| j                   dk7  r| j                  |      }| j                  |      }| j                  |      }| j	                  ||      }|S Nr   )r   r   r   r   r   )rB   r$   r   s      r/   r   zAlignVisionBlock.forward  s[    "
! NN=9M++M: ++M:
MBr.   r   r   s   @r/   r   r   h  s    4'
!'
 '
 	'

 '
 '
 '
 '
 '
 '
R
U%6%6 
5<< 
r.   r   c                   V     e Zd ZdZdef fdZdej                  dee	   de
fdZ xZS )AlignVisionEncoderz
    Forward propagates the embeddings through each vision encoder (EfficientNet) block.

    Args:
        config ([`AlignVisionConfig`]):
            Model configuration class.
    rV   c                     t                    |j                   _         fdt        |j                        }t        fd|j                  D              }d}g }t        |      D ]  }t        ||j                  |         }t        ||j                  |         }|j                  |   }	|j                  |   }
|j                  |   }t         |j                  |               D ]c  }|dk(  }|dkD  rdn|	}	|dkD  r|n|}||j                  v}|j                  |z  |z  }t        ||||	|
||||	      }|j!                  |       |dz  }e  t#        j$                  |       _        y )Nc                 Z    t        t        j                  j                  | z              S r   )r]   mathceildepth_coefficient)repeatsrB   s    r/   round_repeatsz2AlignVisionEncoder.__init__.<locals>.round_repeats  s"    tyy!7!7'!ABCCr.   c              3   .   K   | ]  } |        y wr   r-   )r@   nr   s     r/   rC   z.AlignVisionEncoder.__init__.<locals>.<genexpr>  s     Laq)Ls   r   r   )	rV   r   ru   ro   ra   r   r   r   r   )rs   rt   r   rN   r   sumnum_block_repeatsranger`   r   strideskernel_sizesexpand_ratiosdepthwise_paddingdrop_connect_rater   appendr   
ModuleListblocks)rB   rV   num_base_blocks
num_blockscurr_block_numr   ir   ru   ro   ra   r   jr   r   r   blockr   r   s   `                @r/   rt   zAlignVisionEncoder.__init__  s   !'!9!9	D f001L63K3KLL
' 	$A"66+=+=a+@AF#FF,?,?,BCG^^A&F --a0K!//2L=)A)A!)DEF $q&!e$%Ev!/v7O7O!O"44~E
R	(!!#! +!-'##1
 e$!#'$	$8 mmF+r.   r$   kwargsr;   c                 L    | j                   D ]
  } ||      } t        |      S N)r#   )r   r   )rB   r$   r   r   s       r/   r   zAlignVisionEncoder.forward  s3    
 [[ 	1E!-0M	1 .+
 	
r.   )r%   r&   r'   r(   r   rt   r)   r*   r   r   r   r   r   r   s   @r/   r   r     sC    ),0 ),V

((

 +,

 
(	

r.   r   c                        e Zd ZdZ fdZ	 	 	 	 d
dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  f
d	Z	 xZ
S )AlignTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                        | _	        t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        | j#                  dt%        j&                  |j                        j)                  d      d       | j#                  dt%        j*                  | j,                  j/                         t$        j0                        d       y )	N)padding_idxrq   position_idsr   F)
persistenttoken_type_ids)dtype)rs   rt   r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsr   hidden_dropout_probr   register_bufferr)   rM   r   r   r   sizelongr   s     r/   rt   zAlignTextEmbeddings.__init__  s   !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=ELL)G)GHOOPWXej 	 	
 	ekk$*;*;*@*@*B%**Ubg 	 	
r.   N	input_idsr   r   inputs_embedsr;   c                 6   ||j                         }n|j                         d d }|d   }|| j                  d d d |f   }|st        | d      r-| j                  d d d |f   }|j	                  |d   |      }|}n:t        j                  |t
        j                  | j                  j                        }|| j                  |      }| j                  |      }	||	z   }
| j                  |      }|
|z  }
| j                  |
      }
| j                  |
      }
|
S )Nr   r   r   r   )r   rI   )r  r   hasattrr   r   r)   r   r  rI   r  r	  r  r
  r   )rB   r  r   r   r  input_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr	  r   r  s               r/   r   zAlignTextEmbeddings.forward  s1     #..*K',,.s3K ^
,,Q^<L
 !t-.*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[

SWSdSdSkSk!l  00;M $ : :> J"%::
"66|D))
^^J/
\\*-
r.   )NNNN)r%   r&   r'   r(   rt   r)   
LongTensorr*   r   r   r   r   s   @r/   r   r     s~    Q
$ .2260426&##d*& ((4/& &&-	&
 ((4/& 
&r.   r   modulequerykeyvalueattention_maskscalingr   c                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )NrY   r   r   )r   r   )r   trainingr   )r)   matmul	transposer   rK   softmaxfloat32tor   r   r   
contiguous)
r  r  r  r  r  r  r   r   attn_weightsattn_outputs
             r/   eager_attention_forwardr)  6  s     <<s}}Q':;gEL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r.   c                        e Zd Z fdZ	 ddej
                  dej                  dz  dee   de	ej
                  ej
                  dz  f   fdZ
 xZS )	AlignTextSelfAttentionc                 $   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                         | _        |j                   | _        | j                  dz  | _        y )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g      )rs   rt   r  num_attention_headsr  
ValueErrorrV   r]   attention_head_sizeall_head_sizer   Linearr  r  r  r   attention_probs_dropout_probr   attention_dropoutr  r   s     r/   rt   zAlignTextSelfAttention.__init__M  sC    : ::a?PVXhHi#F$6$6#7 8 445Q8 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF!'!D!D//5r.   Nr$   r  r   r;   c                 x   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  | j                  d|\  }
} |
j                  g |d j!                         }
|
|fS )Nr   r   rY           )r   r  )shaper1  r  viewr"  r  r  r   get_interfacerV   _attn_implementationr)  r   r5  r  reshaper&  )rB   r$   r  r   r  hidden_shapequery_states
key_statesvalue_statesattention_interfacer(  r'  s               r/   r   zAlignTextSelfAttention.forwardb  s>    $))#2.CCbC$*B*BCzz-055lCMMaQRSXXm,11,?II!QO
zz-055lCMMaQRS(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHL((r.   r   )r%   r&   r'   rt   r)   r   r*   r   r   r,   r   r   r   s   @r/   r+  r+  L  sd    60 48)||) ))D0) +,	)
 
u||U\\D00	1)r.   r+  c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )AlignTextSelfOutputc                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                        | _
        y Nr   )rs   rt   r   r3  r  denser
  r  r   r  r   r   s     r/   rt   zAlignTextSelfOutput.__init__  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=r.   r$   input_tensorr;   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   rF  r   r
  rB   r$   rG  s      r/   r   zAlignTextSelfOutput.forward  7    

=1]3}|'CDr.   r%   r&   r'   rt   r)   r   r   r   r   s   @r/   rC  rC    1    >U\\  RWR^R^ r.   rC  c            	            e Zd Z fdZ	 ddej
                  dej                  dz  dee   dej
                  fdZ	 xZ
S )	AlignTextAttentionc                 b    t         |           t        |      | _        t	        |      | _        y r   )rs   rt   r+  rB   rC  outputr   s     r/   rt   zAlignTextAttention.__init__  s&    *62	)&1r.   Nr$   r  r   r;   c                 ^    |} | j                   |fd|i|\  }}| j                  ||      }|S Nr  )rB   rQ  )rB   r$   r  r   residual_s         r/   r   zAlignTextAttention.forward  sK     !$499
)
 
q
 M8<r.   r   )r%   r&   r'   rt   r)   r   r*   r   r   r   r   r   s   @r/   rO  rO    sQ    2 48|| ))D0 +,	
 
r.   rO  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )AlignTextIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )rs   rt   r   r3  r  intermediate_sizerF  rd   r}   strr	   intermediate_act_fnr   s     r/   rt   zAlignTextIntermediate.__init__  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r.   r$   r;   c                 J    | j                  |      }| j                  |      }|S r   )rF  r[  r   s     r/   r   zAlignTextIntermediate.forward  s&    

=100?r.   rL  r   s   @r/   rW  rW    s#    9U\\ ell r.   rW  c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )AlignTextOutputc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        y rE  )rs   rt   r   r3  rY  r  rF  r
  r  r   r  r   r   s     r/   rt   zAlignTextOutput.__init__  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=r.   r$   rG  r;   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   rI  rJ  s      r/   r   zAlignTextOutput.forward  rK  r.   rL  r   s   @r/   r^  r^    rM  r.   r^  c            	            e Zd Z fdZ	 d	dej
                  dej                  dz  dee   dej
                  fdZ	d Z
 xZS )
AlignTextLayerc                     t         |           |j                  | _        d| _        t	        |      | _        t        |      | _        t        |      | _	        y r   )
rs   rt   chunk_size_feed_forwardseq_len_dimrO  	attentionrW  intermediater^  rQ  r   s     r/   rt   zAlignTextLayer.__init__  sI    '-'E'E$+F31&9%f-r.   Nr$   r  r   r;   c                      | j                   |fd|i|}t        | j                  | j                  | j                  |      }|S rS  )rf  r   feed_forward_chunkrd  re  )rB   r$   r  r   s       r/   r   zAlignTextLayer.forward  sY     '
)
 
 2##T%A%A4CSCSUb
 r.   c                 L    | j                  |      }| j                  ||      }|S r   )rg  rQ  )rB   attention_outputintermediate_outputlayer_outputs       r/   ri  z!AlignTextLayer.feed_forward_chunk  s,    "//0@A{{#68HIr.   r   )r%   r&   r'   rt   r)   r   r*   r   r   r   ri  r   r   s   @r/   rb  rb    sV    . 48|| ))D0 +,	
 
$r.   rb  c            	       n     e Zd Z fdZ	 ddej
                  dej                  dz  dee   de	fdZ
 xZS )	AlignTextEncoderc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w )NF)
rs   rt   rV   r   r   r   num_hidden_layersrb  layergradient_checkpointing)rB   rV   r   r   s      r/   rt   zAlignTextEncoder.__init__  sN    ]]E&JbJbDc#dqN6$:#de
&+# $es   A#Nr$   r  r   r;   c                 P    | j                   D ]  } |||fi |} t        |      S r   )rr  r   )rB   r$   r  r   layer_modules        r/   r   zAlignTextEncoder.forward  sC     !JJ 	L( M	 +
 	
r.   r   )r%   r&   r'   rt   r)   r   r*   r   r   r   r   r   r   s   @r/   ro  ro    sM    , 48
||
 ))D0
 +,	

 

r.   ro  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )AlignTextPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )rs   rt   r   r3  r  rF  Tanhr~   r   s     r/   rt   zAlignTextPooler.__init__  s9    YYv1163E3EF
'')r.   r$   r;   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )rF  r~   )rB   r$   first_token_tensorpooled_outputs       r/   r   zAlignTextPooler.forward  s6     +1a40

#566r.   rL  r   s   @r/   rw  rw    s#    $
U\\ ell r.   rw  c                   x     e Zd ZU eed<   dZdZdZ ej                         de
j                  f fd       Z xZS )AlignPreTrainedModelrV   align)imagetextTr  c                 d   t         |   |       t        |t              rt	        j
                  |j                  j                         t	        j                  |j                  j                         t	        j                  |j                  | j                  j                         yt        |t              ryt	        j                  |j                   t#        j$                  |j                   j&                  d         j)                  d             t	        j                  |j*                         yy)zInitialize the weightsr   r   N)rs   _init_weightsrd   
AlignModelinitxavier_uniform_text_projectionweightzeros_rp   	constant_temperaturerV   temperature_init_valuer   copy_r   r)   rM   r8  r   r   )rB   r  r   s     r/   r  z"AlignPreTrainedModel._init_weights  s     	f%fj)  !7!7!>!>?KK..334NN6--t{{/Q/QR 34JJv**ELL9L9L9R9RSU9V,W,^,^_f,ghKK--. 5r.   )r%   r&   r'   r   r+   base_model_prefixinput_modalitiessupports_gradient_checkpointingr)   no_gradr   Moduler  r   r   s   @r/   r~  r~    s@    (&*#U]]_	/BII 	/ 	/r.   r~  zJ
    The text model from ALIGN without any head or projection on top.
    c                   2    e Zd ZU eed<   dZdgZeedZ	ddede
f fdZd Zd Zeee	 	 	 	 	 dd
ej$                  d	z  dej$                  d	z  dej$                  d	z  dej$                  d	z  dej$                  d	z  dee   deez  fd                     Z xZS )AlignTextModelrV   )r  r   )r$   r3   add_pooling_layerc                     t         |   |       || _        t        |      | _        t        |      | _        |rt        |      nd| _        | j                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
rs   rt   rV   r   r   ro  encoderrw  pooler	post_init)rB   rV   r  r   s      r/   rt   zAlignTextModel.__init__2  sM    
 	 -f5'/1Bof- 	r.   c                 .    | j                   j                  S r   r   r  rE   s    r/   get_input_embeddingsz#AlignTextModel.get_input_embeddingsB  s    ...r.   c                 &    || j                   _        y r   r  )rB   r  s     r/   set_input_embeddingsz#AlignTextModel.set_input_embeddingsE  s    */'r.   Nr  r  r   r   r  r   r;   c                    ||t        d      |#| j                  ||       |j                         }n!||j                         dd }nt        d      |\  }}	||j                  n|j                  }
|t	        j
                  ||	f|
      }| j                  ||||      }t        | j                  ||      } | j                  |fd|i|}|d	   }| j                  | j                  |      nd}t        ||
      S )a-  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, AlignTextModel

        >>> model = AlignTextModel.from_pretrained("kakaobrain/align-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("kakaobrain/align-base")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embedsrH   )r  r   r   r  )rV   r  r  r  r   r#   pooler_output)r0  %warn_if_padding_and_no_attention_maskr  rI   r)   onesr   r
   rV   r  r  r   )rB   r  r  r   r   r  r   r  
batch_sizer  rI   embedding_outputencoder_outputssequence_outputr|  s                  r/   r   zAlignTextModel.forwardH  s:   6  ]%>cdd"66y.Q#..*K&',,.s3KTUU!,
J%.%:!!@T@T!"ZZ*j)A6RN??%)'	 + 
 3;;*)
 '$,,
)
 

 *!,8<8OO4UY)-'
 	
r.   TNNNNN)r%   r&   r'   r   r+   r  _no_split_modulesrb  r+  _can_record_outputsr   rt   r  r  r   r   r   r)   r   r   r   r,   r   r   r   r   s   @r/   r  r  $  s      ./',
 4  /0   *..2.2,0-1@
<<$&@
 t+@
 t+	@

 llT)@
 ||d*@
 +,@
 
+	+@
    @
r.   r  zL
    The vision model from ALIGN without any head or projection on top.
    c                        e Zd ZU eed<   dZdZdZdZdgZ	de
iZdef fdZeee	 ddej"                  d	z  d
ee   deez  fd                     Z xZS )AlignVisionModelrV   r   )r  Frx   r   r$   c                    t         |   |       || _        t        |      | _        t        |      | _        |j                  dk(  r't        j                  |j                  d      | _        nN|j                  dk(  r't        j                  |j                  d      | _        nt        d|j                         | j                          y )NmeanT)	ceil_moder\   z2config.pooling must be one of ['mean', 'max'] got )rs   rt   rV   rh   r   r   r  pooling_typer   	AvgPool2d
hidden_dimr  	MaxPool2dr0  poolingr  r   s     r/   rt   zAlignVisionModel.__init__  s     /7)&1 &(,,v'8'8DIDK  E),,v'8'8DIDKQRXR`R`Qabcc 	r.   Nr   r;   c                     |t        d      | j                  |      } | j                  |fi |}|d   }| j                  |      }|j	                  |j
                  dd       }t        ||      S )a  
        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, AlignVisionModel

        >>> model = AlignVisionModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```Nz You have to specify pixel_valuesr   rY   r  )r0  r   r  r  r<  r8  r   )rB   r   r   r  r  r#   r|  s          r/   r   zAlignVisionModel.forward  s    < ?@@??<8&$,,

 ,A.$56%--m.A.A"1.EF7/'
 	
r.   r   )r%   r&   r'   r   r+   main_input_namer  r  _input_embed_layerr  r   r  rt   r   r   r   r)   r*   r   r   r,   r   r   r   r   s   @r/   r  r    s     $O!&+#&+,)0 "   26*
''$.*
 +,*
 
9	9	*
    *
r.   r  c                   H    e Zd ZU eed<   def fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eez  fd              Zeedej                  d	e
e   d
eez  fd              Zee	 	 	 	 	 	 	 ddej"                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	e
e   d
eez  fd              Z xZS )r  rV   c                    t         |   |       t        |j                  t              s"t        dt        |j                         d      t        |j                  t              s"t        dt        |j                         d      |j                  }|j                  }|j                  | _	        |j                  | _        t        |      | _        t        |      | _        t!        j"                  | j                  | j                        | _        t!        j&                  t)        j*                  | j,                  j.                              | _        | j3                          y )NzLconfig.text_config is expected to be of type AlignTextConfig but is of type .zPconfig.vision_config is expected to be of type AlignVisionConfig but is of type )rs   rt   rd   text_configr   	TypeErrortypevision_configr   projection_dimr  text_embed_dimr  
text_modelr  vision_modelr   r3  r  	Parameterr)   tensorrV   r  r  r  )rB   rV   r  r  r   s       r/   rt   zAlignModel.__init__  s#    &,,o>++,-Q0 
 &..0AB--./q2 
 ((,,$33)55(5,];!yy)<)<d>Q>QR<<T[[5W5W(XY 	r.   Nr  r  r   r   r  r   r;   c           	           | j                   d|||||d|}|d   dddddf   }| j                  |      |_        |S )a  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoTokenizer, AlignModel

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("kakaobrain/align-base")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     text_features = model.get_text_features(**inputs)
        ```r  r  r   r   r  r   Nr-   )r  r  r  )	rB   r  r  r   r   r  r   text_outputsr#   s	            r/   get_text_featureszAlignModel.get_text_features  sd    2 4C4?? 4
))%'4
 4
 )OAq!G4%)%9%9:K%L"r.   r   c                 *     | j                   dd|i|S )a}  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AlignModel
        >>> from transformers.image_utils import load_image

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(images=image, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     image_features = model.get_image_features(**inputs)
        ```r   r-   )r  )rB   r   r   s      r/   get_image_featureszAlignModel.get_image_features'  s    . !t  ElEfEEr.   return_lossc           	          | j                   dd|i|}	 | j                  d|||||d|}
|	d   }|
d   dddddf   }| j                  |      }||j                  ddd	      z  }||j                  ddd	      z  }t	        j
                  ||j                               | j                  z  }|j                         }d}|rt        |      }t        ||||||
|	
      S )a  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AlignModel
        >>> from transformers.image_utils import load_image

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(
        ...     images=image, text=["a photo of a cat", "a photo of a dog"], return_tensors="pt", padding=True
        ... )

        >>> with torch.inference_mode():
        ...     outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```r   r  r   r   NrY   r   T)r   r   keepdim)r6   r7   r8   r2   r"   r9   r:   r-   )
r  r  r  normr)   r!  rR   r  rU   r5   )rB   r  r   r  r   r   r  r  r   vision_outputsr  r"   r2   r8   r7   r6   s                   r/   r   zAlignModel.forward@  s(   N +** 
%


 't 
))%'
 
 &a("1oaAg.**;7 $l&7&7!T&7&RR!K$4$4qb$$4$OO  ,,{LNN4DEHXHXX*,,.o.D-+#%* .
 	
r.   r  )NNNNNNN)r%   r&   r'   r   r+   rt   r   r   r)   r   r   r   r,   r   r  r*   r  r  r   r5   r   r   r   s   @r/   r  r    s   { <  *..2.2,0-1"<<$&" t+" t+	"
 llT)" ||d*" +," 
+	+"  "H F!--F9?@R9SF	+	+F  F.  .215.2.2,0-1#'K
##d*K
 ''$.K
 t+	K

 t+K
 llT)K
 ||d*K
 D[K
 +,K
 
	K
  K
r.   r  )r~  r  r  r  r  )r7  )Ur(   r   collections.abcr   dataclassesr   typingr   r)   r    r   r  activationsr	   masking_utilsr
   modeling_layersr   modeling_outputsr   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   configuration_alignr   r   r   
get_loggerr%   loggerr!   r1   r5   r   rO   rU   r]   r`   r,   r   rf   r  rh   rw   r   r   r   r   r   r   r   r   r   r)  r+  rC  rO  rW  r^  rb  ro  rw  r~  r  r  r  __all__r-   r.   r/   <module>r     s`     $ !    & ! 6 9  G & 6 M M I 5 P P 
		H	% 
 :[ : : 
 	7; 	7 	7 
 
+  
   
JuU\\ uell u-5<< -ELL -+ 3  @S5[ @$ @*BII 4
 
6		 6$		 $P$BII $N BNryy Nb>
 >
B9")) 9F %II%<<% 
% <<	%
 LL4'% % %,3)RYY 3)n"))  .BII  bii / B
ryy 
4bii  /? / /& 
b
) b

b
J 
I
+ I

I
X m
% m
 m
` Wr.   