^jDNdZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z ddlmZdd lmZdd lmZmZmZdd lmZdd lmZd dlmZmZmZmZd dlm Z m!Z!m"Z"d dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1m2Z2m3Z3edeGdde!Z4edeGdde"Z5edeGdde Z6Gdde(Z7Gdde/Z8GddeZ9Gd d!e3Z:Gd"d#eZ;Gd$d%e0Z<Gd&d'e1Z=Gd(d)eZ>Gd*d+e&Z?Gd,d-e2Z@Gd.d/e%ZAGd0d1e$ZBGd2d3e'ZCGd4d5e&ZDGd6d7e*ZEGd8d9e)ZFGd:d;e+ZGed<=Gd>d?eFZHGd@dAeFZIGdBdCeeFZJgdDZKy)EzPyTorch AltCLIP model.N)strict)initialization)create_bidirectional_mask)BaseModelOutputWithPooling'BaseModelOutputWithPoolingAndProjection)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)merge_with_config_defaults)capture_outputs)ChineseCLIPModelChineseCLIPTextAttentionChineseCLIPTextLayerChineseCLIPTextSelfAttention) CLIPConfigCLIPTextConfigCLIPVisionConfig) CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayer CLIPOutputCLIPPreTrainedModelCLIPVisionEmbeddingsCLIPVisionModel_get_vector_normimage_text_contrastive_loss)RobertaEmbeddingsRobertaIntermediate RobertaOutput RobertaPoolerRobertaSelfOutputz BAAI/AltCLIP) checkpointceZdZUdZdZeed<dZeed<dZeed<dZ eed <d Z eed <d Z e ed <dZ eezed<dZeezed<dZeed<dZeed<dZeed<dZedzed<dZedzed<dZedzed<dZeed<eZeZy)AltCLIPTextConfigaR project_dim (`int`, *optional*, defaults to 768): The dimensions of the teacher model before the mapping layer. Examples: ```python >>> from transformers import AltCLIPTextModel, AltCLIPTextConfig >>> # Initializing a AltCLIPTextConfig with BAAI/AltCLIP style configuration >>> configuration = AltCLIPTextConfig() >>> # Initializing a AltCLIPTextModel (with random weights) from the BAAI/AltCLIP style configuration >>> model = AltCLIPTextModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```i vocab_sizei hidden_sizenum_hidden_layersnum_attention_headsiintermediate_sizegelu hidden_actg?hidden_dropout_probrattention_probs_dropout_probimax_position_embeddingstype_vocab_sizeg{Gz?initializer_factorN pad_token_id bos_token_idr eos_token_id project_dim)__name__ __module__ __qualname____doc__r*int__annotations__r+r-r/r0r2strr3floatr4r5r7r8r9r:r;r=AttributeErrorprojection_dimattention_dropoutv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/altclip/modular_altclip.pyr)r):s &JKs!!!s!J'*u*01 #+1#&S&OS $$ L#*  L#*  L#* K#%N&(rJr)ceZdZdZy)AltCLIPVisionConfiga Example: ```python >>> from transformers import AltCLIPVisionConfig, AltCLIPVisionModel >>> # Initializing a AltCLIPVisionConfig with BAAI/AltCLIP style configuration >>> configuration = AltCLIPVisionConfig() >>> # Initializing a AltCLIPVisionModel (with random weights) from the BAAI/AltCLIP style configuration >>> model = AltCLIPVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```N)r>r?r@rArIrJrKrMrMcs rJrMc eZdZUdZdZeed<y) AltCLIPConfiga Example: ```python >>> from transformers import AltCLIPConfig, AltCLIPModel >>> # Initializing a AltCLIPConfig with BAAI/AltCLIP style configuration >>> configuration = AltCLIPConfig() >>> # Initializing a AltCLIPModel (with random weights) from the BAAI/AltCLIP style configuration >>> model = AltCLIPModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a AltCLIPConfig from a AltCLIPTextConfig and a AltCLIPVisionConfig >>> # Initializing a AltCLIPText and AltCLIPVision configuration >>> config_text = AltCLIPTextConfig() >>> config_vision = AltCLIPVisionConfig() >>> config = AltCLIPConfig(text_config=config_text, vision_config=config_vision) ```r<rGN)r>r?r@rArGrBrCrIrJrKrOrOws 0NCrJrOc eZdZy) AltCLIPOutputNr>r?r@rIrJrKrQrQrJrQc eZdZy)AltRobertaEmbeddingsNrRrIrJrKrUrUrSrJrUceZdZfdZxZS)AltRobertaSelfAttentionc2t||d|_y)NF)super__init__ is_causalselfconfig __class__s rKrZz AltRobertaSelfAttention.__init__s  rJr>r?r@rZ __classcell__r_s@rKrWrWs rJrWc eZdZy)AltRobertaSelfOutputNrRrIrJrKrdrdrSrJrdceZdZfdZxZS)AltRobertaAttentioncbt|t||_t ||_yN)rYrZrWr]rdoutputr\s rKrZzAltRobertaAttention.__init__s& +F3 *62 rJr`rbs@rKrfrfs 33rJrfc eZdZy)AltRobertaIntermediateNrRrIrJrKrkrkrSrJrkc eZdZy)AltRobertaOutputNrRrIrJrKrmrmrSrJrmceZdZfdZxZS)AltRobertaLayerct|t||_t ||_t ||_yrh)rYrZrf attentionrk intermediatermrir\s rKrZzAltRobertaLayer.__init__s3 ,V426:&v. rJr`rbs@rKroros //rJroc(eZdZdZdeffd ZxZS)AltRobertaEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`AltRobertaEncoderLayer`]. Args: config: AltCLIPTextConfig r^ct||tjt |j Dcgc] }t |c}|_ycc}wrh)rYrZnn ModuleListranger-rolayers)r]r^_r_s rKrZzAltRobertaEncoder.__init__s>  mmeFLdLdFe$f_V%<$fg $fsA)r>r?r@rAr)rZrarbs@rKrtrtsh0hhrJrtc eZdZy)AltRobertaPoolerNrRrIrJrKr|r|rSrJr|c eZdZy)AltCLIPAttentionNrRrIrJrKr~r~rSrJr~c eZdZy) AltCLIPMLPNrRrIrJrKrrrSrJrc$eZdZdeffd ZxZS)AltCLIPEncoderLayerr^c$t||yrh)rYrZr\s rKrZzAltCLIPEncoderLayer.__init__s  rJ)r>r?r@rMrZrarbs@rKrrs!2!!rJrc eZdZy)AltCLIPEncoderNrRrIrJrKrrrSrJrc eZdZy)AltCLIPVisionEmbeddingsNrRrIrJrKrrrSrJrcLeZdZgdZeedZejdZ y)AltCLIPPreTrainedModel)rUrorr hidden_states attentionsctj|||jj}t |t rt j|jd|jdz|zt j|jj|jj|zt j|jj|jj|zt j|jt!j"|j$j'dyt |t(r|jdzd|jj*zdzz|z}|jdz|z}t j|j,j|t j|j.j|t j|j0j|t j|j2j|yt |t4r|jj6dzd|jj*zdzz|z}d|jj6zdz|z}t j|j8j|t j|j:j|yt |t<rwt j|j>j|j@dz|zt j|jBj|jDdz|zyt |tFryt j|jt!j"|jjHdj'dt jJ|jLyy) zInitialize the weightsgg)meanstd)r)r6rrN)'r _init_weightsr^r8 isinstancerinitnormal_class_embedding embed_dimpatch_embeddingweightinitializer_rangeposition_embeddingcopy_ position_idstorcharange num_positionsexpandr~r-q_projk_projv_projout_projrr+fc1fc2 AltCLIPModeltext_projectiontext_embed_dimvisual_projectionvision_embed_dimrUshapezeros_token_type_ids)r]modulefactor in_proj_std out_proj_stdfc_stds rKrz$AltCLIPPreTrainedModel._init_weightss %%dF3// f5 6 LL//cv?O?OQU?UX^?^ _ LL//66FMM<[<[^dr?r@_no_split_modulesrr~_can_record_outputsrno_gradrrIrJrKrrs2u,& U]]_ / /rJrceZdZfdZxZS)AltCLIPVisionModelc "t|di|S)a Examples: ```python >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> from transformers import AutoProcessor, AltCLIPVisionModel >>> model = AltCLIPVisionModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```rI)rYforwardr] super_kwargsr_s rKrzAltCLIPVisionModel.forwards.w...rJ)r>r?r@rrarbs@rKrrs //rJraE The model behaves as an encoder following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762 ) custom_introceZdZUeed<dZdZeedZ dfd Z e e e ddejdzdejdzd ejdzd ejdzd ejdzd eed eezfdZxZS)AltRobertaModelr^textword_embeddingsrct||t||_t ||_|r t |nd|_|jy)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) rYrZrU embeddingsrtencoderr|pooler post_init)r]r^add_pooling_layerr_s rKrZzAltRobertaModel.__init__>sE  .v6(0 2C&v.  rJN input_idsattention_maskrr inputs_embedskwargsreturnc |du|duz r td|j||||}t|j||}|j|fd|i|}|d}|j |j |nd} t || S)aK Examples: ```python >>> from transformers import AutoTokenizer, AltRobertaModel >>> model = AltRobertaModel.from_pretrained("openai/alt_roberta-vit-base-patch32") >>> tokenizer = AutoTokenizer.from_pretrained("openai/alt_roberta-vit-base-patch32") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled (EOS token) states ```Nz:You must specify exactly one of input_ids or inputs_embeds)rrrr)r^rrrr)last_hidden_state pooler_output) ValueErrorrrr^rrr) r]rrrrrrencoder_outputssequence_output pooled_outputs rKrzAltRobertaModel.forwardJs6  -t"; <YZ Z%)' ( 3;;')  '$,,  )   *!,8< 8O O4UY )-'  rJ)TNNNNN)r>r?r@r)rCinput_modalities_input_embed_layerrorWrrZrrr rTensorr r tuplerrrarbs@rKrr+s  *(-  *..2.2,0-1 3 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 +,3  + +3  3 rJrceZdZUeed<dZdZdZfdZe e dde jdzde jdzd e jdzd e jdzd e jdzd e ed eezfdZxZS)AltCLIPTextModelr^rrrobertac&t||t|d|_t j |j |j|_t j|j |j|_ |jy)NF)r)eps) rYrZrrrvLinearr+r=transformation LayerNormlayer_norm_epspre_LNrr\s rKrZzAltCLIPTextModel.__init__se  &vG  ii(:(:F>> from transformers import AutoProcessor, AltCLIPTextModel >>> model = AltCLIPTextModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> texts = ["it's a cat", "it's a dog"] >>> inputs = processor(text=texts, padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```)rrrrrrN)rrrrrI)rrrrrr) r]rrrrrroutputsrprojection_staters rKrzAltCLIPTextModel.forwards:$,, ))%'    "!*++o6 ..?(A. 6.'!//))   rJr)r>r?r@r)rCrrbase_model_prefixrZr r rrr r rrrrarbs@rKrrs  *!*..2.2,0-1 3 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 +,3  8 83 3 rJrceZdZUeed<deffd ZfdZfdZ ddejdzdejdzdejdzd ejdzd ejdzd e dzd e d e edeezfdZxZS)rr^ct|||j}|j|_t j |jj|_tj |jj|_ yrh) rYrZ text_configr=rr _from_configr^ text_modelr vision_config vision_model)r]r^rr_s rKrZzAltCLIPModel.__init__sb  (( )55*77 8O8OP.;;DKK>> import torch >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> with torch.inference_mode(): ... text_features = model.get_text_features(**inputs) ```rI)rYget_text_featuresrs rKrzAltCLIPModel.get_text_featuressw(8<88rJc "t|di|S)ao Examples: ```python >>> import torch >>> from transformers import AutoProcessor, AltCLIPModel >>> from transformers.image_utils import load_image >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```rI)rYget_image_featuresrs rKrzAltCLIPModel.get_image_featuress&w)9L99rJNr pixel_valuesrrr return_lossinterpolate_pos_encodingrrc 2|jd||d|} |jd||||d|} | d} |j| } | d} |j| } | t | z } | t | z } t j | | jj| j} | |jjj| jz} | j}d}|r t| }t||| | | | | S)u return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. Examples: ```python >>> import torch >>> from transformers import AutoProcessor, AltCLIPModel >>> from transformers.image_utils import load_image >>> model = AltCLIPModel.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") >>> processor = AutoProcessor.from_pretrained("OFA-Sys/chinese-clip-vit-base-patch16") >>> url = "https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg" >>> image = load_image(url) >>> inputs = processor(text=["杰尼龟", "妙蛙种子", "小火龙", "皮卡丘"], images=image, return_tensors="pt", padding=True) >>> with torch.inference_mode(): ... outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities ```)rr)rrrrr6N)losslogits_per_imagelogits_per_text text_embeds image_embedstext_model_outputvision_model_outputrI)rrrrr rmatmulttodevice logit_scaleexpr!rQ)r]rrrrrrrrvision_outputs text_outputsrrrrrs rKrzAltCLIPModel.forwardsGF+** %%=   't ))%    &a( --l; "1o **;7 $&6|&DD !$4[$AA  ,,{LNN4D4G4G HZHZ4[\)D,<,<,@,@,B,E,EkFXFX,YY*,,. .?D-+#%* .  rJ)NNNNNNF)r>r?r@rOrCrZrrr LongTensor FloatTensorrboolr r rrQrrarbs@rKrrs W}W9":..215.2.204#').L ##d*L ''$.L  t+ L  t+ L &&- L D[L #'L +,L   L rJr)rrrrr)rMrO)LrArtorch.nnrvhuggingface_hub.dataclassesrrr masking_utilsrmodeling_outputsrrmodeling_utilsr processing_utilsr utilsr r r utils.genericrutils.output_capturingr"chinese_clip.modeling_chinese_cliprrrrclip.configuration_cliprrrclip.modeling_cliprrrrrrrrr r!roberta.modeling_robertar"r#r$r%r&r)rMrOrQrUrWrdrfrkrmrortr|r~rrrrrrrrr__all__rIrJrKrs .&6.&II75 SR   >*$)$)+$)N>* * + $>*J+8 J  , :  , 323 0  } /*/ h h }  }   !*!  [  2 (/0(/V//6L ,L L ^B -B J| #%;| ~ rJ