Ë
    ÛÓ^j«  ã                   ó2  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* e e G d„ de«      «       «       Z+ G d„ dejX                  «      Z-	 dNdejX                  dej\                  dej\                  dej\                  dej\                  dz  de/de/fd„Z0 G d„ d ejX                  «      Z1 G d!„ d"ejX                  «      Z2 G d#„ d$ejX                  «      Z3 G d%„ d&ejX                  «      Z4 G d'„ d(ejX                  «      Z5 G d)„ d*e«      Z6 G d+„ d,ejX                  «      Z7 G d-„ d.ejX                  «      Z8 G d/„ d0ejX                  «      Z9 G d1„ d2ejX                  «      Z: G d3„ d4e«      Z; G d5„ d6ejX                  «      Z< G d7„ d8ejX                  «      Z=e  G d9„ d:e«      «       Z> e d;¬<«       G d=„ d>e>«      «       Z? e d?¬<«       G d@„ dAe>«      «       Z@ G dB„ dCe>«      ZAdDej\                  dEej\                  fdF„ZBdGej\                  dEej\                  fdH„ZCdIej\                  dEej\                  fdJ„ZDe  G dK„ dLe>«      «       ZEg dM¢ZFy)Oé    )ÚCallable)Ú	dataclass)ÚAnyNé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚ'BaseModelOutputWithPoolingAndProjection)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚAltCLIPConfigÚAltCLIPTextConfigÚAltCLIPVisionConfigc                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   dZeed<   dZeed	<   d
ee   fd„Zy)ÚAltCLIPOutputaµ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`].
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`AltCLIPTextModel`].
    vision_model_output (`BaseModelOutputWithPooling`):
        The output of the [`AltCLIPVisionModel`].
    NÚlossÚlogits_per_imageÚlogits_per_textÚtext_embedsÚimage_embedsÚtext_model_outputÚvision_model_outputÚreturnc                 óB   — t        d„ | j                  «       D «       «      S )Nc              3   ó`   K  — | ]&  }t        |t        «      r|j                  «       n|–— Œ( y ­w©N)Ú
isinstancer   Úto_tuple)Ú.0Úvs     úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/altclip/modeling_altclip.pyú	<genexpr>z)AltCLIPOutput.to_tuple.<locals>.<genexpr>H   s$   è ø€ Ò^È1¤Z°´;Ô%?�Q—Z‘Z”\ÀQÓFÑ^ùs   ‚,.)ÚtupleÚvalues)Úselfs    r.   r+   zAltCLIPOutput.to_tupleG   s   € ÜÑ^ÐPT×P[ÑP[ÓP]Ô^Ó^Ð^ó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r    r!   r"   r#   r$   r   r%   r0   r   r+   © r3   r.   r   r   )   s�   … ñð& &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø15Ð�e×'Ñ'¨$Ñ.Ó5Ø04€O�U×&Ñ&¨Ñ-Ó4Ø,0€K�×"Ñ" TÑ)Ó0Ø-1€L�%×#Ñ# dÑ*Ó1Ø48ÐÐ1Ó8Ø6:ÐÐ3Ó:ð_˜% ™*ô _r3   r   c                   óì   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  ded	ej                  fd
„Z
ed„ «       Zedd„«       Zˆ xZS )ÚAltRobertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óT  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j
                  |j                  ¬«      | _
        t        j                  |j                  «      | _        | j                  dt!        j"                  |j$                  «      j'                  d«      d¬«       | j                  dt!        j(                  | j*                  j-                  «       t         j.                  ¬«      d¬«       |j                  | _        t        j                  |j$                  |j
                  | j0                  ¬«      | _        y )	N)Úpadding_idx©ÚepsÚposition_ids©r   éÿÿÿÿF©Ú
persistentÚtoken_type_ids©Údtype)ÚsuperÚ__init__ÚnnÚ	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr8   ÚarangeÚmax_position_embeddingsÚexpandÚzerosrB   ÚsizeÚlongr?   Úposition_embeddings©r2   ÚconfigÚ	__class__s     €r.   rK   zAltRobertaEmbeddings.__init__N   s4  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒà×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	ô 	
ð "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ r3   NÚ	input_idsrG   rB   Úinputs_embedsÚpast_key_values_lengthr&   c                 ó  — |€<|�| j                  || j                  |«      }n| j                  || j                  «      }|�|j                  «       }n|j                  «       d d }|\  }}|€³t	        | d«      rm| j
                  j                  |j                  «      j                  |j                  d   d«      }	t        j                  |	d|¬«      }	|	j                  ||«      }n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j!                  |«      }
||
z   }| j#                  |«      }||z   }| j%                  |«      }| j'                  |«      }|S )NrD   rG   r   r   )ÚdimÚindex©rI   Údevice)Ú"create_position_ids_from_input_idsr?   Ú&create_position_ids_from_inputs_embedsr^   ÚhasattrrG   Útork   r\   Úshaper8   Úgatherr]   r_   rB   rQ   rS   r`   rT   rX   )r2   rd   rG   rB   re   rf   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsrS   Ú
embeddingsr`   s                r.   ÚforwardzAltRobertaEmbeddings.forwardb   s€  € ð ÐØÐ$à#×FÑFØ˜t×/Ñ/Ð1Gó ‘ð  $×JÑJÈ=ÐZ^×ZjÑZjÓk�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
�Jð
 Ð!Ü�tÐ-Ô.à*.×*=Ñ*=×*@Ñ*@À×ATÑATÓ*U×*\Ñ*\Ð]i×]oÑ]oÐpqÑ]rÐtvÓ*wÐ'Ü*/¯,©,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
ÈJÓ!W‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ"5Ñ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr3   c                 óæ   — | j                  «       dd }|d   }t        j                  |dz   ||z   dz   t        j                  | j                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        NrD   r   rj   r   )r^   r8   rZ   r_   rk   Ú	unsqueezer\   )re   r?   rr   Úsequence_lengthrB   s        r.   rm   z;AltRobertaEmbeddings.create_position_ids_from_inputs_embeds’   sp   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÄeÇjÁjÐYf×YmÑYmô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r3   c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r   ©rh   )ÚneÚintr8   ÚcumsumÚtype_asr_   )rd   r?   rf   ÚmaskÚincremental_indicess        r.   rl   z7AltRobertaEmbeddings.create_position_ids_from_input_ids¤   sW   € ð �|‰|˜KÓ(×,Ñ,Ó.ˆÜ$Ÿ|™|¨D°aÔ8×@Ñ@ÀÓFÐI_Ñ_ÐcgÑgÐØ"×'Ñ'Ó)¨KÑ7Ð7r3   )NNNNr   )r   )r4   r5   r6   r7   rK   r8   Ú
LongTensorr9   r~   ÚTensorrw   Ústaticmethodrm   rl   Ú__classcell__©rc   s   @r.   r=   r=   K   s´   ø„ ÙQô
ð, .2Ø26Ø04Ø26Ø&'ñ.à×#Ñ# dÑ*ð.ð ×(Ñ(¨4Ñ/ð.ð ×&Ñ&¨Ñ-ð	.ð
 ×(Ñ(¨4Ñ/ð.ð !$ð.ð 
�‰ó.ð` ñ=ó ð=ð" ò8ó ô8r3   r=   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrX   c                 óÀ  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |dt         j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Né   r   rD   )rh   rI   )ÚpÚtrainingr   )r8   ÚmatmulÚ	transposerL   Ú
functionalÚsoftmaxÚfloat32ro   rI   rX   r‘   Ú
contiguous)
rˆ   r‰   rŠ   r‹   rŒ   r�   rX   ÚkwargsÚattn_weightsÚattn_outputs
             r.   Úeager_attention_forwardr›   µ   sº   € ô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r3   c                   ó¦   ‡ — e Zd Zˆ fd„Z	 ddej
                  dej                  dz  dee   de	ej
                  ej
                  dz  f   fd„Z
ˆ xZS )	ÚAltRobertaSelfAttentionc                 ó2  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                   «      | _        |j                   | _        | j                  dz  | _        d| _        y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)ç      à¿F)rJ   rK   rO   Únum_attention_headsrn   Ú
ValueErrorrb   r~   Úattention_head_sizeÚall_head_sizerL   ÚLinearr‰   rŠ   r‹   rV   Úattention_probs_dropout_probrX   Úattention_dropoutr�   Ú	is_causalra   s     €r.   rK   z AltRobertaSelfAttention.__init__Ì   sJ  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 ˆŒØ#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒØ!'×!DÑ!DˆÔØ×/Ñ/°Ñ5ˆŒØˆ�r3   NÚhidden_statesrŒ   r˜   r&   c                 óx  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  sdn| j                  | j                  dœ|¤Ž\  }
} |
j                  g |¢d‘­Ž j!                  «       }
|
|fS )NrD   r   r�   ç        )rX   r�   )rp   r¤   r‰   Úviewr“   rŠ   r‹   r   Úget_interfacerb   Ú_attn_implementationr›   r‘   r¨   r�   Úreshaper—   )r2   rª   rŒ   r˜   rr   Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerš   r™   s               r.   rw   zAltRobertaSelfAttention.forwardâ   s>  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆà—z‘z -Ó0×5Ñ5°lÓC×MÑMÈaÐQRÓSˆØ—X‘X˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆ
Ø—z‘z -Ó0×5Ñ5°lÓC×MÑMÈaÐQRÓSˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(r3   r)   )r4   r5   r6   rK   r8   r„   r9   r   r   r0   rw   r†   r‡   s   @r.   r�   r�   Ë   sd   ø„ ôð2 48ñ)à—|‘|ð)ð ×)Ñ)¨DÑ0ð)ð Ð+Ñ,ð	)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷)r3   r�   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr@   )rJ   rK   rL   r¦   rO   ÚdenserT   rU   rV   rW   rX   ra   s     €r.   rK   zAltRobertaSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r3   rª   Úinput_tensorr&   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r)   ©rº   rX   rT   ©r2   rª   r»   s      r.   rw   zAltRobertaSelfOutput.forward	  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr3   ©r4   r5   r6   rK   r8   r„   rw   r†   r‡   s   @r.   r·   r·     ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r3   r·   c            	       ó‚   ‡ — e Zd Zˆ fd„Z	 ddej
                  dej                  dz  dee   dej
                  fd„Z	ˆ xZ
S )	ÚAltRobertaAttentionc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y r)   )rJ   rK   r�   r2   r·   Úoutputra   s     €r.   rK   zAltRobertaAttention.__init__  s&   ø€ Ü‰ÑÔÜ+¨FÓ3ˆŒ	Ü*¨6Ó2ˆ�r3   Nrª   rŒ   r˜   r&   c                 ó^   — |} | j                   |fd|i|¤Ž\  }}| j                  ||«      }|S ©NrŒ   )r2   rÅ   ©r2   rª   rŒ   r˜   ÚresidualÚ_s         r.   rw   zAltRobertaAttention.forward  sK   € ð !ˆØ$˜4Ÿ9™9Øñ
à)ð
ð ñ
Ñˆ�qð
 Ÿ™ M°8Ó<ˆØÐr3   r)   )r4   r5   r6   rK   r8   r„   r9   r   r   rw   r†   r‡   s   @r.   rÃ   rÃ     sQ   ø„ ô3ð 48ñà—|‘|ðð ×)Ñ)¨DÑ0ðð Ð+Ñ,ð	ð
 
�‰÷r3   rÃ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r)   )rJ   rK   rL   r¦   rO   Úintermediate_sizerº   r*   Ú
hidden_actÚstrr   Úintermediate_act_fnra   s     €r.   rK   zAltRobertaIntermediate.__init__'  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r3   rª   r&   c                 óJ   — | j                  |«      }| j                  |«      }|S r)   )rº   rÑ   ©r2   rª   s     r.   rw   zAltRobertaIntermediate.forward/  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr3   rÀ   r‡   s   @r.   rÌ   rÌ   &  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r3   rÌ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r¹   )rJ   rK   rL   r¦   rÎ   rO   rº   rT   rU   rV   rW   rX   ra   s     €r.   rK   zAltRobertaOutput.__init__6  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r3   rª   r»   r&   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r)   r½   r¾   s      r.   rw   zAltRobertaOutput.forward<  r¿   r3   rÀ   r‡   s   @r.   rÕ   rÕ   5  rÁ   r3   rÕ   c            	       óˆ   ‡ — e Zd Zˆ fd„Z	 d	dej
                  dej                  dz  dee   dej
                  fd„Z	d„ Z
ˆ xZS )
ÚAltRobertaLayerc                 ó²   •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        t        |«      | _        t        |«      | _	        y )Nr   )
rJ   rK   Úchunk_size_feed_forwardÚseq_len_dimrÃ   Ú	attentionrÌ   ÚintermediaterÕ   rÅ   ra   s     €r.   rK   zAltRobertaLayer.__init__D  sI   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ,¨VÓ4ˆŒÜ2°6Ó:ˆÔÜ& vÓ.ˆ�r3   Nrª   rŒ   r˜   r&   c                 óˆ   —  | j                   |fd|i|¤Ž}t        | j                  | j                  | j                  |«      }|S rÇ   )rÝ   r   Úfeed_forward_chunkrÛ   rÜ   )r2   rª   rŒ   r˜   s       r.   rw   zAltRobertaLayer.forwardL  sY   € ð '˜Ÿ™Øñ
à)ð
ð ñ
ˆô 2Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUbó
ˆð Ðr3   c                 óL   — | j                  |«      }| j                  ||«      }|S r)   )rÞ   rÅ   )r2   Úattention_outputÚintermediate_outputÚlayer_outputs       r.   rà   z"AltRobertaLayer.feed_forward_chunk^  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr3   r)   )r4   r5   r6   rK   r8   r„   r9   r   r   rw   rà   r†   r‡   s   @r.   rÙ   rÙ   C  sV   ø„ ô/ð 48ñà—|‘|ðð ×)Ñ)¨DÑ0ðð Ð+Ñ,ð	ð
 
�‰óö$r3   rÙ   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dz  dee	   de
fd„Zˆ xZS )
ÚAltRobertaEncoderzº
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`AltRobertaEncoderLayer`].

    Args:
        config: AltCLIPTextConfig
    rb   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
rJ   rK   rb   rL   Ú
ModuleListÚrangeÚnum_hidden_layersrÙ   ÚlayersÚgradient_checkpointing©r2   rb   rÊ   rc   s      €r.   rK   zAltRobertaEncoder.__init__m  sN   ø€ Ü‰ÑÔØˆŒÜ—m‘mÄeÈF×LdÑLdÓFeÖ$fÀ¤_°VÕ%<Ò$fÓgˆŒØ&+ˆÕ#ùò %gó   ½A#NrŒ   r˜   r&   c                 óT   — |}| j                   D ]  } |||fi |¤Ž}Œ t        |¬«      S ©N)Úlast_hidden_state©rì   r   ©r2   re   rŒ   r˜   rª   Úencoder_layers         r.   rw   zAltRobertaEncoder.forwards  óH   € ð &ˆØ!Ÿ[™[ò 	ˆMÙ)ØØñð ñ‰Mð	ô Ø+ô
ð 	
r3   r)   )r4   r5   r6   r7   r   rK   r8   r„   r   r   r   rw   r†   r‡   s   @r.   ræ   ræ   d  sL   ø„ ñð,Ð0õ ,ð /3ñ
ð Ÿ™ tÑ+ð
ð Ð+Ñ,ð	
ð
 
÷
r3   ræ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r)   )rJ   rK   rL   r¦   rO   rº   ÚTanhÚ
activationra   s     €r.   rK   zAltRobertaPooler.__init__‡  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r3   rª   r&   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rº   rû   )r2   rª   Úfirst_token_tensorÚpooled_outputs       r.   rw   zAltRobertaPooler.forwardŒ  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr3   rÀ   r‡   s   @r.   rø   rø   †  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r3   rø   c                   ó¶   ‡ — e Zd ZdZdeez  fˆ fd„Z	 d
dej                  dej                  dz  de	e
   deej                  ej                  dz  f   fd	„Zˆ xZS )ÚAltCLIPAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrb   c                 ó€  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  dz  | _        |j                  | _
        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nr¡   F)rJ   rK   rb   rO   Ú	embed_dimr¢   Ú	num_headsÚhead_dimÚscaler¨   rX   r©   rL   r¦   Úk_projÚv_projÚq_projÚout_projra   s     €r.   rK   zAltCLIPAttention.__init__˜  sÏ   ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ—]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r3   Nrª   rŒ   r˜   r&   c                 ó¦  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      }| j                  |«      }| j	                  |«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  | j                  sdn| j                  dœ|¤Ž\  }
} |
j                  g |¢d‘­Ž j!                  «       }
| j#                  |
«      }
|
|fS )z#Input shape: Batch x Time x ChannelNrD   r   r�   r¬   )r�   rX   )rp   r  r  r  r  r­   r“   r   r®   rb   r¯   r›   r  r‘   rX   r°   r—   r	  )r2   rª   rŒ   r˜   rr   r±   ÚqueriesÚkeysr1   rµ   rš   r™   s               r.   rw   zAltCLIPAttention.forward§  sO  € ð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,ˆØ�{‰{˜=Ó)ˆØ—‘˜]Ó+ˆà—,‘,˜|Ó,×6Ñ6°q¸!Ó<ˆØ�y‰y˜Ó&×0Ñ0°°AÓ6ˆØ—‘˜\Ó*×4Ñ4°Q¸Ó:ˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð —J‘JØ#Ÿ}š}‘C°$·,±,ñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r3   r)   )r4   r5   r6   r7   r   r   rK   r8   r„   r   r   r0   rw   r†   r‡   s   @r.   r   r   •  su   ø„ ÙGðBÐ2Ð5FÑFõ Bð$ /3ñ%)à—|‘|ð%)ð Ÿ™ tÑ+ð%)ð Ð+Ñ,ð	%)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷%)r3   r   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
AltCLIPMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y r)   )rJ   rK   rb   r   rÏ   Úactivation_fnrL   r¦   rO   rÎ   Úfc1Úfc2ra   s     €r.   rK   zAltCLIPMLP.__init__Ð  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r3   rª   r&   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r)   )r  r  r  rÓ   s     r.   rw   zAltCLIPMLP.forward×  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr3   rÀ   r‡   s   @r.   r  r  Ï  s$   ø„ ôKð U§\¡\ð °e·l±l÷ r3   r  c                   ó~   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dee   dej                  fd„Z
ˆ xZS )ÚAltCLIPEncoderLayerrb   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y r¹   )rJ   rK   rO   r  r   Ú	self_attnrL   rT   rU   Úlayer_norm1r  ÚmlpÚlayer_norm2ra   s     €r.   rK   zAltCLIPEncoderLayer.__init__ß  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ)¨&Ó1ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ˜fÓ%ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕr3   rª   rŒ   r˜   r&   c                 ó¸   — |}| j                  |«      } | j                  d||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)rª   rŒ   r;   )r  r  r  r  rÈ   s         r.   rw   zAltCLIPEncoderLayer.forwardç  s‚   € ð !ˆà×(Ñ(¨Ó7ˆØ)˜4Ÿ>™>ð 
Ø'Ø)ñ
ð ñ
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆàÐr3   )r4   r5   r6   r   rK   r8   r„   r   r   r9   rw   r†   r‡   s   @r.   r  r  Þ  sQ   ø„ ðSÐ2õ Sðà—|‘|ðð Ÿ™ðð Ð+Ñ,ð	ð
 
×	Ñ	÷r3   r  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dz  dee	   de
fd„Zˆ xZS )
ÚAltCLIPEncoderz³
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`AltCLIPEncoderLayer`].

    Args:
        config: AltCLIPConfig
    rb   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w rè   )
rJ   rK   rb   rL   ré   rê   rë   r  rì   rí   rî   s      €r.   rK   zAltCLIPEncoder.__init__  sP   ø€ Ü‰ÑÔØˆŒÜ—m‘mÌ%ÐPV×PhÑPhÓJiÖ$jÀQÔ%8¸Õ%@Ò$jÓkˆŒØ&+ˆÕ#ùò %krï   NrŒ   r˜   r&   c                 óT   — |}| j                   D ]  } |||fi |¤Ž}Œ t        |¬«      S rñ   ró   rô   s         r.   rw   zAltCLIPEncoder.forward  rö   r3   r)   )r4   r5   r6   r7   r   rK   r8   r„   r   r   r   rw   r†   r‡   s   @r.   r  r  ÿ  sK   ø„ ñð,˜}õ ,ð /3ñ
ð Ÿ™ tÑ+ð
ð Ð+Ñ,ð	
ð
 
÷
r3   r  c                   óž   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zd
dej                  dej                  fd	„Z
ˆ xZS )ÚAltCLIPVisionEmbeddingsrb   c                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasr�   r   rB   rC   rE   )rJ   rK   rb   rO   r  Ú
image_sizeÚ
patch_sizerL   Ú	Parameterr8   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsrM   Úposition_embeddingrY   rZ   r\   ra   s     €r.   rK   z AltCLIPVisionEmbeddings.__init__"  s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕpr3   rv   ÚheightÚwidthr&   c                 óÒ  — |j                   d   dz
  }| j                  j                  j                  d«      }|j                   d   dz
  }t        j
                  j                  «       s%||k(  r ||k(  r| j                  | j                  «      S |dd…dd…f   }|dd…dd…f   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz  «      }|j                  d|||	«      }|j                  dddd«      }t        j                  j                  ||
|fdd	¬
«      }|j                  dddd«      j                  dd|	«      }t	        j                   ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   NrD   ç      à?r   r�   ÚbicubicF)r^   ÚmodeÚalign_cornersr|   )rp   r2  Úweightry   r8   ÚjitÚ
is_tracingrB   r)  r   r°   ÚpermuterL   r”   Úinterpolater­   Úcat)r2   rv   r3  r4  r0  r2  r1  Úclass_pos_embedÚpatch_pos_embedrh   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r.   Úinterpolate_pos_encodingz0AltCLIPVisionEmbeddings.interpolate_pos_encoding8  sv  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr3   Úpixel_valuesc                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (z).rH   r�   r   rD   r|   )rp   r(  r£   r/  r:  rI   ro   Úflattenr“   r,  r\   r8   r?  rE  r2  rB   )r2   rF  rE  rs   rÊ   r3  r4  Útarget_dtypeÚpatch_embedsÚclass_embedsrv   s              r.   rw   zAltCLIPVisionEmbeddings.forwarda  s6  € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uÙ'¨V°t·±Ò-FÈ%ÐSW×SbÑSbÒJbÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐK\Ð\]Ð^b×^mÑ^mÐ]nÐnpÐqóð ð ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJØÐr3   ©F)r4   r5   r6   r   rK   r8   r„   r~   rE  r9   rw   r†   r‡   s   @r.   r!  r!  !  se   ø„ ðqÐ2õ qð,'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]ió 'DñR E×$5Ñ$5ð ÐZ_×ZfÑZf÷ r3   r!  c                   ó€   ‡ — e Zd ZU eed<   dZdZg d¢ZdZdZ	dZ
dZdZeedœZ ej"                  «       ˆ fd„«       Zˆ xZS )ÚAltCLIPPreTrainedModelrb   Úaltclip)ÚimageÚtext)r=   rÙ   r  r!  T©rª   Ú
attentionsc                 ó  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      �rt        j                  |j                  d|j                  dz  |z  ¬«       t        j                  |j                  j                  |j                  j                  |z  ¬«       t        j                  |j                  j                  |j                  j                  |z  ¬«       t        j                  |j                  t!        j"                  |j$                  «      j'                  d«      «       yt	        |t(        «      rî|j                  dz  d|j                  j*                  z  dz  z  |z  }|j                  dz  |z  }t        j                  |j,                  j                  |¬«       t        j                  |j.                  j                  |¬«       t        j                  |j0                  j                  |¬«       t        j                  |j2                  j                  |¬«       yt	        |t4        «      r¯|j                  j6                  dz  d|j                  j*                  z  dz  z  |z  }d|j                  j6                  z  dz  |z  }t        j                  |j8                  j                  |¬«       t        j                  |j:                  j                  |¬«       yt	        |t<        «      rwt        j                  |j>                  j                  |j@                  dz  |z  ¬«       t        j                  |jB                  j                  |jD                  dz  |z  ¬«       yt	        |tF        «      ryt        j                  |j                  t!        j"                  |j                  jH                  d   «      j'                  d«      «       t        jJ                  |jL                  «       yy)	zInitialize the weightsr¬   r¡   )ÚmeanÚstd)rW  rC   r�   rD   N)'rJ   Ú_init_weightsrb   Úinitializer_factorr*   r!  ÚinitÚnormal_r,  r  r/  r:  Úinitializer_ranger2  Úcopy_rB   r8   rZ   r1  r\   r   rë   r  r  r  r	  r  rO   r  r  ÚAltCLIPModelÚtext_projectionÚtext_embed_dimÚvisual_projectionÚvision_embed_dimr=   rp   Úzeros_rG   )r2   rˆ   ÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdrc   s         €r.   rX  z$AltCLIPPreTrainedModel._init_weights…  sÚ  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fÔ5Õ6Ü�L‰L˜×/Ñ/°c¸v×?OÑ?OÐQUÑ?UÐX^Ñ?^Õ_Ü�L‰L˜×/Ñ/×6Ñ6¸F¿M¹M×<[Ñ<[Ð^dÑ<dÕeÜ�L‰L˜×2Ñ2×9Ñ9¸v¿}¹}×?^Ñ?^ÐagÑ?gÕhÜ�J‰J�v×*Ñ*¬E¯L©L¸×9MÑ9MÓ,N×,UÑ,UÐV]Ó,^Õ_Ü˜Ô 0Ô1Ø!×+Ñ+¨TÑ1°q¸6¿=¹=×;ZÑ;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"×,Ñ,¨dÑ2°fÑ<ˆLÜ�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×/Ñ/°\ÖBÜ˜¤
Ô+Ø!Ÿ=™=×4Ñ4°dÑ:ÀÀFÇMÁM×DcÑDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&Ÿ-™-×3Ñ3Ñ3¸Ñ<¸vÑEˆFÜ�L‰L˜Ÿ™×*Ñ*°Õ7Ü�L‰L˜Ÿ™×*Ñ*°Ö<Ü˜¤Ô-Ü�L‰LØ×&Ñ&×-Ñ-Ø×)Ñ)¨4Ñ/°&Ñ8õô �L‰LØ×(Ñ(×/Ñ/Ø×+Ñ+¨TÑ1°FÑ:öô ˜Ô 4Ô5Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�K‰K˜×-Ñ-Õ.ð 6r3   )r4   r5   r6   r   r:   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendr  r   Ú_can_record_outputsr8   Úno_gradrX  r†   r‡   s   @r.   rO  rO  t  sb   ø… àÓØ!ÐØ(ÐÚuÐà&*Ð#Ø€NØÐØÐØ"&Ðà,Ø&ñÐð
 €U‡]�]ƒ_ó /ó ô /r3   rO  zN
    The vision model from ALTCLIP without any head or projection on top.
    )Úcustom_introc                   ó¬   ‡ — e Zd ZU eed<   dZdZdZdefˆ fd„Ze	 e
d¬«      e	 	 ddej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚAltCLIPVisionModelrb   rF  )rQ  r/  c                 ó4  •— t         ‰| �  |«       |j                  }t        |«      | _        t        j                  ||j                  ¬«      | _        t        |«      | _
        t        j                  ||j                  ¬«      | _        | j                  «        y r¹   )rJ   rK   rO   r!  rv   rL   rT   rU   Úpre_layrnormr  ÚencoderÚpost_layernormÚ	post_init)r2   rb   r  rc   s      €r.   rK   zAltCLIPVisionModel.__init__´  so   ø€ Ü‰Ñ˜Ô Ø×&Ñ&ˆ	ä1°&Ó9ˆŒÜŸL™L¨¸×8MÑ8MÔNˆÔÜ% fÓ-ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÔØ�‰Õr3   F)Útie_last_hidden_statesNrE  r˜   r&   c                 óÞ   — | j                  ||¬«      }| j                  |«      } | j                  dd|i|¤Ž}|j                  }|dd…ddd…f   }| j	                  |«      }t        ||¬«      S )a  
        Examples:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> from transformers import AutoProcessor, AltCLIPVisionModel

        >>> model = AltCLIPVisionModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```)rE  re   Nr   ©rò   Úpooler_outputr;   )rv   rv  rw  rò   rx  r   )r2   rF  rE  r˜   rª   Úencoder_outputsrò   rþ   s           r.   rw   zAltCLIPVisionModel.forward¾  sˆ   € ð> Ÿ™¨ÐOg˜ÓhˆØ×)Ñ)¨-Ó8ˆà+7¨4¯<©<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
r3   rè   )r4   r5   r6   r   r:   Úmain_input_nameri  Ú_input_embed_layerrK   r   r   r   r8   r9   Úboolr   r   r   rw   r†   r‡   s   @r.   rt  rt  ©  s•   ø… ð  ÓØ$€OØ!ÐØ*ÐðÐ2õ ð  Ù¨EÔ2Øð 26Ø05ñ+
à×'Ñ'¨$Ñ.ð+
ð #'¨¡+ð+
ð Ð+Ñ,ð	+
ð
 
$ò+
ó ó 3ó  ô+
r3   rt  aE  
    The model behaves as an encoder following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    c                   ó  ‡ — e Zd ZU eed<   dZdZeedœZ	dˆ fd„	Z
eee	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dee   deez  fd„«       «       «       Zˆ xZS )ÚAltRobertaModelrb   ©rR  rQ   rS  c                 ó¬   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        |rt        |«      nd| _        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)	rJ   rK   r=   rv   ræ   rw  rø   Úpoolerry  )r2   rb   Úadd_pooling_layerrc   s      €r.   rK   zAltRobertaModel.__init__  sE   ø€ ô
 	‰Ñ˜Ô Ü.¨vÓ6ˆŒÜ(¨Ó0ˆŒÙ2CÔ& vÔ.ÈˆŒà�‰Õr3   Nrd   rŒ   rG   rB   re   r˜   r&   c                 ó  — |du |duz  rt        d«      ‚| j                  ||||¬«      }t        | j                  ||¬«      } | j                  |fd|i|¤Ž}|d   }| j
                  �| j                  |«      nd}	t        ||	¬«      S )aK  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, AltRobertaModel

        >>> model = AltRobertaModel.from_pretrained("openai/alt_roberta-vit-base-patch32")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/alt_roberta-vit-base-patch32")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```Nz:You must specify exactly one of input_ids or inputs_embeds)rd   rB   rG   re   )rb   re   rŒ   rŒ   r   r|  )r£   rv   r	   rb   rw  r†  r   )
r2   rd   rŒ   rG   rB   re   r˜   r~  Úsequence_outputrþ   s
             r.   rw   zAltRobertaModel.forward  sº   € ð6 ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàŸ™ØØ%Ø)Ø'ð	 (ó 
ˆô 3Ø—;‘;Ø'Ø)ô
ˆð '˜$Ÿ,™,Øñ
à)ð
ð ñ
ˆð
 *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆä)Ø-Ø'ô
ð 	
r3   )T©NNNNN)r4   r5   r6   r   r:   ri  r€  rÙ   r�   rp  rK   r   r   r   r8   r„   r   r   r0   r   rw   r†   r‡   s   @r.   rƒ  rƒ  ï  sÜ   ø… ð ÓØ Ðà*Ðà(Ø-ñÐõ

ð  ØØð *.Ø.2Ø.2Ø,0Ø-1ñ3
à—<‘< $Ñ&ð3
ð Ÿ™ tÑ+ð3
ð Ÿ™ tÑ+ð	3
ð
 —l‘l TÑ)ð3
ð —|‘| dÑ*ð3
ð Ð+Ñ,ð3
ð 
Ð+Ñ	+ò3
ó ó ó  ô3
r3   rƒ  c                   ó  ‡ — e Zd ZU eed<   dZdZdZˆ fd„Ze	e
	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚAltCLIPTextModelrb   r„  rQ   Úrobertac                 ó&  •— t         ‰| �  |«       t        |d¬«      | _        t	        j
                  |j                  |j                  «      | _        t	        j                  |j                  |j                  ¬«      | _        | j                  «        y )NF)r‡  r@   )rJ   rK   rƒ  r�  rL   r¦   rO   Úproject_dimÚtransformationrT   rU   Úpre_LNry  ra   s     €r.   rK   zAltCLIPTextModel.__init__M  se   ø€ Ü‰Ñ˜Ô Ü& vÀÔGˆŒÜ Ÿi™i¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜ—l‘l 6×#5Ñ#5¸6×;PÑ;PÔQˆŒØ�‰Õr3   Nrd   rŒ   rG   rB   re   r˜   r&   c           	      óØ   —  | j                   d|||||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }	|	dd…df   }
t        |	|
|j                  |j
                  ¬«      S )a+  
        Examples:

        ```python
        >>> from transformers import AutoProcessor, AltCLIPTextModel

        >>> model = AltCLIPTextModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> texts = ["it's a cat", "it's a dog"]

        >>> inputs = processor(text=texts, padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```)rd   rŒ   rG   rB   re   r   N)rò   r}  rª   rT  r;   )r�  r‘  r�  r   rª   rT  )r2   rd   rŒ   rG   rB   re   r˜   Úoutputsr‰  Úprojection_stater}  s              r.   rw   zAltCLIPTextModel.forwardT  s“   € ð: �$—,‘,ð 
ØØ)Ø)Ø%Ø'ñ
ð ñ
ˆð " !™*ˆð Ÿ+™+ oÓ6ˆð  ×.Ñ.¨Ó?ÐØ(ª¨A¨Ñ.ˆä6Ø.Ø'Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   rŠ  )r4   r5   r6   r   r:   ri  r€  rh  rK   r   r   r8   r„   r   r   r0   r   rw   r†   r‡   s   @r.   rŒ  rŒ  G  sÊ   ø… ØÓØ ÐØ*ÐØ!Ðôð Øð *.Ø.2Ø.2Ø,0Ø-1ñ3
à—<‘< $Ñ&ð3
ð Ÿ™ tÑ+ð3
ð Ÿ™ tÑ+ð	3
ð
 —l‘l TÑ)ð3
ð —|‘| dÑ*ð3
ð Ð+Ñ,ð3
ð 
Ð8Ñ	8ò3
ó ó ô3
r3   rŒ  Úlogitsr&   c                 ó’   — t         j                  j                  | t        j                  t        | «      | j                  ¬«      «      S )N)rk   )rL   r”   Úcross_entropyr8   rZ   Úlenrk   )r•  s    r.   Úcontrastive_lossr™  Ž  s/   € Ü�=‰=×&Ñ& v¬u¯|©|¼CÀ»KÐPV×P]ÑP]Ô/^Ó_Ð_r3   Ú
similarityc                 óR   — t        | «      }t        | j                  «      }||z   dz  S )Ng       @)r™  ÚT)rš  Úcaption_lossÚ
image_losss      r.   Úimage_text_contrastive_lossrŸ  ’  s*   € Ü# JÓ/€LÜ! *§,¡,Ó/€JØ˜:Ñ%¨Ñ,Ð,r3   Útensorc                 óŽ   — t        j                  | d«      }t        j                  |dd¬«      }t        j                  |d«      }|S )z½
    This method is equivalent to tensor.norm(p=2, dim=-1, keepdim=True) and used to make
    model `executorch` exportable. See issue https://github.com/pytorch/executorch/issues/3566
    r�   rD   T)rh   Úkeepdimr6  )r8   ÚpowÚsum)r   Úsquare_tensorÚ
sum_tensorÚnormed_tensors       r.   Ú_get_vector_normr¨  ˜  s<   € ô
 —I‘I˜f aÓ(€MÜ—‘˜=¨b¸$Ô?€JÜ—I‘I˜j¨#Ó.€MØÐr3   c                   ó  ‡ — e Zd ZU eed<   defˆ fd„Zee	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  de
e   d	eez  fd
„«       «       Zee	 ddej                  dede
e   d	eez  fd„«       «       Zee	 	 	 	 	 	 	 ddej$                  dz  dej                  dz  dej                  dz  dej                  dz  dej$                  dz  dedz  dede
e   d	eez  fd„«       «       Zˆ xZS )r^  rb   c                 óØ  •— t         ‰| �  |«       |j                  }|j                  }|j                  | _        |j
                  | _        |j                  | _        t        j                  | j                  j                  «      | _        t        j                  | j                  j                  «      | _        t        j                   | j                  | j                  d¬«      | _        t        j                   | j                  | j                  d¬«      | _        t        j&                  t)        j*                  | j                  j,                  «      «      | _        | j1                  «        y )NF)r'  )rJ   rK   Útext_configÚvision_configÚprojection_dimr�  r`  rO   rb  rŒ  Ú_from_configrb   Ú
text_modelrt  Úvision_modelrL   r¦   ra  r_  r*  r8   r   Úlogit_scale_init_valueÚlogit_scalery  )r2   rb   r«  r¬  rc   s       €r.   rK   zAltCLIPModel.__init__§  sý   ø€ Ü‰Ñ˜Ô Ø×(Ñ(ˆØ×,Ñ,ˆà$×3Ñ3ˆÔØ)×5Ñ5ˆÔØ -× 9Ñ 9ˆÔÜ*×7Ñ7¸¿¹×8OÑ8OÓPˆŒÜ.×;Ñ;¸D¿K¹K×<UÑ<UÓVˆÔä!#§¡¨4×+@Ñ+@À$×BUÑBUÐ\aÔ!bˆÔÜ!Ÿy™y¨×)<Ñ)<¸d×>QÑ>QÐX]Ô^ˆÔÜŸ<™<¬¯©°T·[±[×5WÑ5WÓ(XÓYˆÔð 	�‰Õr3   Nrd   rŒ   rG   rB   r˜   r&   c                 óŒ   —  | j                   d||||dœ|¤Ž}|j                  dd…ddd…f   }| j                  |«      |_        |S )að  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AltCLIPModel

        >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     text_features = model.get_text_features(**inputs)
        ```©rd   rŒ   rG   rB   Nr   r;   )r¯  rò   r_  r}  )r2   rd   rŒ   rG   rB   r˜   Útext_outputsrþ   s           r.   Úget_text_featureszAltCLIPModel.get_text_features¹  sa   € ð0 4C°4·?±?ð 4
ØØ)Ø)Ø%ñ	4
ð
 ñ4
ˆð %×6Ñ6²q¸!ºQ°wÑ?ˆØ%)×%9Ñ%9¸-Ó%HˆÔ"àÐr3   rF  rE  c                 óv   —  | j                   d||ddœ|¤Ž}|j                  }| j                  |«      |_        |S )ao  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AltCLIPModel
        >>> from transformers.image_utils import load_image

        >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(images=image, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     image_features = model.get_image_features(**inputs)
        ```T)rF  rE  Úreturn_dictr;   )r°  r}  ra  )r2   rF  rE  r˜   Úvision_outputsrþ   s         r.   Úget_image_featureszAltCLIPModel.get_image_featuresÝ  sU   € ð4 6G°T×5FÑ5Fð 6
Ø%Ø%=Øñ6
ð ñ	6
ˆð '×4Ñ4ˆØ'+×'=Ñ'=¸mÓ'LˆÔ$àÐr3   Úreturn_lossc           	      ó2  —  | j                   d||dœ|¤Ž}	 | j                  d||||dœ|¤Ž}
|	d   }| j                  |«      }|
d   }| j                  |«      }|t	        |«      z  }|t	        |«      z  }t        j                  ||j                  «       j                  |j                  «      «      }|| j                  j                  «       j                  |j                  «      z  }|j                  «       }d}|rt        |«      }t        ||||||
|	¬«      S )u  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AltCLIPModel
        >>> from transformers.image_utils import load_image

        >>> model = AltCLIPModel.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16")
        >>> processor = AutoProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16")

        >>> url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg"
        >>> image = load_image(url)

        >>> inputs = processor(text=["æ�°å°¼é¾Ÿ", "å¦™è›™ç§�å­�", "å°�ç�«é¾™", "çš®å�¡ä¸˜"], images=image, return_tensors="pt", padding=True)

        >>> with torch.inference_mode():
        ...     outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```)rF  rE  r´  r   N)r   r    r!   r"   r#   r$   r%   r;   )r°  r¯  ra  r_  r¨  r8   r’   Útro   rk   r²  ÚexprŸ  r   )r2   rd   rF  rŒ   rG   rB   r»  rE  r˜   r¹  rµ  r#   r"   r!   r    r   s                   r.   rw   zAltCLIPModel.forward  sG  € ðJ +˜×*Ñ*ð 
Ø%Ø%=ñ
ð ñ
ˆð '�t—‘ð 
ØØ)Ø)Ø%ñ	
ð
 ñ
ˆð & aÑ(ˆØ×-Ñ-¨lÓ;ˆà" 1‘oˆØ×*Ñ*¨;Ó7ˆð $Ô&6°|Ó&DÑDˆØ!Ô$4°[Ó$AÑAˆô  Ÿ,™, {°L·N±NÓ4D×4GÑ4GÈ×HZÑHZÓ4[Ó\ˆØ)¨D×,<Ñ,<×,@Ñ,@Ó,B×,EÑ,EÀk×FXÑFXÓ,YÑYˆØ*×,Ñ,Ó.ÐàˆÙÜ.¨Ó?ˆDäØØ-Ø+Ø#Ø%Ø*Ø .ô
ð 	
r3   )NNNrM  )NNNNNNF)r4   r5   r6   r   r:   rK   r   r   r8   r„   r   r   r0   r   r¶  r9   r�  rº  rƒ   r   rw   r†   r‡   s   @r.   r^  r^  £  sÔ  ø… àÓð˜}õ ð$ Øð /3Ø.2Ø,0ñ à—<‘<ð ð Ÿ™ tÑ+ð ð Ÿ™ tÑ+ð	 ð
 —l‘l TÑ)ð ð Ð+Ñ,ð ð 
Ð+Ñ	+ò ó ó ð ðD Øð */ñ!à×'Ñ'ð!ð #'ð!ð Ð+Ñ,ð	!ð
 
Ð+Ñ	+ò!ó ó ð!ðF Øð .2Ø15Ø.2Ø.2Ø04Ø#'Ø).ñL
à×#Ñ# dÑ*ðL
ð ×'Ñ'¨$Ñ.ðL
ð Ÿ™ tÑ+ð	L
ð
 Ÿ™ tÑ+ðL
ð ×&Ñ&¨Ñ-ðL
ð ˜D‘[ðL
ð #'ðL
ð Ð+Ñ,ðL
ð 
�Ñ	òL
ó ó ôL
r3   r^  )rO  rt  rŒ  r^  )r¬   )GÚcollections.abcr   Údataclassesr   Útypingr   r8   Útorch.nnrL   Ú r   rZ  Úactivationsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_altclipr   r   r   r   ÚModuler=   r„   Úfloatr›   r�   r·   rÃ   rÌ   rÕ   rÙ   ræ   rø   r   r  r  r  r!  rO  rt  rƒ  rŒ  r™  rŸ  r¨  r^  Ú__all__r;   r3   r.   ú<module>rÒ     s®  ðõ( %Ý !Ý ã Ý å &Ý !Ý 6Ý 9ß tÑ tß FÝ &Ý 6ß aÕ aÝ 7Ý 5ß XÑ Xð Ø
ô_�Kó _ó ó ð_ô@g8˜2Ÿ9™9ô g8ðb ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ô,4)˜bŸi™iô 4)ôn˜2Ÿ9™9ô ô˜"Ÿ)™)ô ô,˜RŸY™Yô ô�r—y‘yô ôÐ0ô ôB
˜Ÿ	™	ô 
ôD�r—y‘yô ô7)�r—y‘yô 7)ôt�—‘ô ôÐ4ô ôB
�R—Y‘Yô 
ôDP˜bŸi™iô Pðf ô1/˜_ó 1/ó ð1/ñh ðôô
>
Ð/ó >
óð
>
ñB ðôôL
Ð,ó L
óðL
ô^B
Ð-ô B
ðN`˜UŸ\™\ð `¨e¯l©ló `ð-¨E¯L©Lð -¸U¿\¹\ó -ð˜UŸ\™\ð ¨e¯l©ló ð ôl
Ð)ó l
ó ðl
ò^ _�r3   