^jgRdZddlZddlZddlmZddlmZddlZddlm Z ddlm Z ddl m Z dd lmZdd lmZdd lmZdd lmZmZmZdd lmZmZddlmZddlmZmZm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,ede GddeZ-ede Gdde Z.ede GddeZ/Gdde'Z0eeGddeZ1eeGddeZ2Gd d!e+Z3Gd"d#e)Z4Gd$d%e#Z5Gd&d'e"Z6Gd(d)e%Z7Gd*d+e%Z8eGd,d-e(Z9Gd.d/e$Z:Gd0d1e9Z;Gd2d3e*Z<Gd4d5e,Z=Gd6d7e&Z>ed89Gd:d;e9Z?gd<Z@y)=zPyTorch CLIPSeg model.N) dataclass)Any)strict)nn)initialization)BaseModelOutputWithPooling)PreTrainedModel)Unpack) ModelOutputTransformersKwargsauto_docstring)can_return_tuplemerge_with_config_defaults)capture_outputs) CLIPConfigCLIPTextConfigCLIPVisionConfig) CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayer CLIPModel CLIPOutputCLIPPreTrainedModelCLIPTextEmbeddings CLIPTextModelCLIPVisionEmbeddingsCLIPVisionModelzCIDAS/clipseg-rd64) checkpointceZdZdZeZy)CLIPSegTextConfiga Example: ```python >>> from transformers import CLIPSegTextConfig, CLIPSegTextModel >>> # Initializing a CLIPSegTextConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegTextConfig() >>> # Initializing a CLIPSegTextModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegTextModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```N__name__ __module__ __qualname____doc__AttributeErrorprojection_dimv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/clipseg/modular_clipseg.pyr#r#0 $%Nr,r#ceZdZdZeZy)CLIPSegVisionConfiga Example: ```python >>> from transformers import CLIPSegVisionConfig, CLIPSegVisionModel >>> # Initializing a CLIPSegVisionConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegVisionConfig() >>> # Initializing a CLIPSegVisionModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```Nr$r+r,r-r0r0Fr.r,r0ceZdZUdZdZeeeedfzed<dZ eed<dZ eed<d Z e ezed <d Z eed <d Zeed<dZeed<dZeed<y) CLIPSegConfiga extract_layers (`list[int]`, *optional*, defaults to `[3, 6, 9]`): Layers to extract when forwarding the query image through the frozen visual backbone of CLIP. reduce_dim (`int`, *optional*, defaults to 64): Dimensionality to reduce the CLIP vision embedding. conditional_layer (`int`, *optional*, defaults to 0): The layer to use of the Transformer encoder whose activations will be combined with the condition embeddings using FiLM (Feature-wise Linear Modulation). If 0, the last layer is used. use_complex_transposed_convolution (`bool`, *optional*, defaults to `False`): Whether to use a more complex transposed convolution in the decoder, enabling more fine-grained segmentation.. Example: ```python >>> from transformers import CLIPSegConfig, CLIPSegModel >>> # Initializing a CLIPSegConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegConfig() >>> # Initializing a CLIPSegModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a CLIPSegConfig from a CLIPSegTextConfig and a CLIPSegVisionConfig >>> # Initializing a CLIPSegText and CLIPSegVision configuration >>> config_text = CLIPSegTextConfig() >>> config_vision = CLIPSegVisionConfig() >>> config = CLIPSegConfig(text_config=config_text, vision_config=config_vision) ```)r .extract_layers@ reduce_dimdecoder_num_attention_headsdecoder_attention_dropout quick_geludecoder_hidden_actidecoder_intermediate_sizerconditional_layerF"use_complex_transposed_convolutionN)r%r&r'r(r5listinttuple__annotations__r7r9r;floatr=strr>r?r@boolr+r,r-r2r2\su! F3M5**C/047>7;Je'',-4;r,rMceZdZUdZdZej dzed<dZej dzed<dZ ej dzed<dZ ej dzed<dZ e ed<dZ eed<d eefd Zy) CLIPSegImageSegmentationOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Binary cross entropy loss for segmentation. logits (`torch.FloatTensor` of shape `(batch_size, height, width)`): Classification scores for each pixel. conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Conditional embeddings used for segmentation. pooled_output (`torch.FloatTensor` of shape `(batch_size, embed_dim)`): Pooled output of the [`CLIPSegVisionModel`]. vision_model_output (`BaseModelOutputWithPooling`): The output of the [`CLIPSegVisionModel`]. decoder_output (`CLIPSegDecoderOutput`): The output of the [`CLIPSegDecoder`]. NlossrNconditional_embeddings pooled_outputvision_model_outputdecoder_outputreturncBtd|jDS)Nc3`K|]&}t|tr|jn|(ywN) isinstancer to_tuple).0vs r- z:CLIPSegImageSegmentationOutput.to_tuple..s$^1Z;%?QZZ\QF^s,.)rCvalues)selfs r-r_z'CLIPSegImageSegmentationOutput.to_tuples^PTP[P[P]^^^r,)r%r&r'r(rUrQrRrDrNrVrWrXr rYrMrCrr_r+r,r-rTrTs &*D%  d ")'+FE   $+7;E--4;.2M5$$t+26:3:+/N(/_%*_r,rTcReZdZddejdej ffd ZxZS)CLIPSegVisionEmbeddings pixel_valuesrZc&t|||yr]superforward)rdrginterpolate_pos_encoding __class__s r-rkzCLIPSegVisionEmbeddings.forwards  &>?r,T)r%r&r'rQrRTensorrk __classcell__rms@r-rfrfs,@E$5$5@Y^YeYe@@r,rfc eZdZy)CLIPSegTextEmbeddingsNrJr+r,r-rsrsrKr,rsc eZdZy)CLIPSegAttentionNrJr+r,r-rururKr,ruc eZdZy) CLIPSegMLPNrJr+r,r-rwrwrKr,rwc eZdZy)CLIPSegEncoderLayerNrJr+r,r-ryryrKr,rycfeZdZdZdej dej deejfdZy)CLIPSegDecoderLayerz CLIPSeg decoder layer, which is identical to `CLIPSegEncoderLayer`, except that normalization is applied after self-attention/MLP, rather than before. rOattention_maskrZc |}|jd||d|\}}||z}|j|}|}|j|}||z}|j|}|S)N)rOr|r+) self_attn layer_norm1mlp layer_norm2)rdrOr|kwargsresidual_s r-rkzCLIPSegDecoderLayer.forwards !)4>> ')   q !=0 ((7  /  =0 ((7 r,N) r%r&r'r(rQrorCrRrkr+r,r-r{r{s; ||  u  ! r,r{cHeZdZeegedZejdZ y)CLIPSegPreTrainedModel)rOrPctj|||jj}t |t rt j|jjd|dzt j|jjd|dzt j|jtj|jjdj!dy t |t"rt j|j$d|j&dz|zt j|j(j|jj*|zt j|jj|jj*|zt j|jtj|j,j!dy t |t.r|j&dzd|jj0zdzz|z}|j&dz|z}t j|j2j|t j|j4j|t j|j6j|t j|j8j|y t |t:r|jj<dzd|jj0zdzz|z}d|jj<zdz|z}t j|j>j|t j|j@j|y t |tBrwt j|jDj|jFdz|zt j|jHj|jJdz|zy y ) zInitialize the weightsr:g{Gz?)meanstd)rg)rrN)&r _init_weightsconfiginitializer_factorr^rsinitnormal_token_embeddingweightposition_embeddingcopy_ position_idsrQarangeshapeexpandrfclass_embedding embed_dimpatch_embeddinginitializer_range num_positionsrunum_hidden_layersq_projk_projv_projout_projrw hidden_sizefc1fc2 CLIPSegModeltext_projectiontext_embed_dimvisual_projectionvision_embed_dim)rdmodulefactor in_proj_std out_proj_stdfc_stds r-rz$CLIPSegPreTrainedModel._init_weightss  %%dF3// f3 4 LL//66Sftm T LL2299RV W JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h  7 8 LL//cv?O?OQU?UX^?^ _ LL//66FMM<[<[^dintermediate_size hidden_actr{layers post_init)rdrtransposed_kernelsdepthrdecoder_configrms r-rzCLIPSegDecoder.__init__&s[  !'!9!9 &"7"79J9JK  &"7"79J9JK  4 4"("6"6"A"AQ"FH\H\HgHgklHl!m *,-- &++V->->AWXY ""%%%%* 21 5-a0   ""%%*A;Ma;PYklmYn +D '+-*<*<!!1f&:&:&E&EfNbNbNmNm+D 'F))*}}UZ[`Ua bPQRYYv++779J9J K b  v';';<%+%6%6"-3-O-O*+1+K+K($*!mmRWX[\b\q\qXrRs$tQ%8%H$tu   c%us ;L LrOrVrrZc |ddd}d}tt||j|jD]\}\}}} | | ||z}n| |}||jk(rJ|j ||j dddz|j|z}|j ddd}||fddi|}|ddddddfjdd}ttj|jd} |jd} |j| |jd| | }|j|jd} t!| S)a/ conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*): The conditional embeddings for the query images. If provided, the model will use this instead of computing the embeddings from the conditional_pixel_values. Nrrrrr|)rN) enumerateziprrr?rpermuter transposerBmathsqrtrviewrsqueezerM) rdrOrVr activationsoutputi activationlayerreducesize batch_sizerNs r-rkzCLIPSegDecoder.forwardRsk$DbD) .7KVZVbVb8c.d B *A* E6! +f4 +D***'=>PQSTVWAXX[_[h[h*\ 1a06A$A&AF B12q!++Aq1499V\\!_-.+11!4 Za$E,,V4<>> from transformers import AutoTokenizer, CLIPSegTextModel >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegTextModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled (EOS token) states ```r+rird super_kwargsrms r-rkzCLIPSegTextModel.forward~s w...r,)r%r&r'rCr rkrprqs@r-rr}s/1K)K//r,rc `eZdZ ddejdzdedzdeedee zffd Z xZ S)CLIPSegVisionModelrgNrlrrZc &t|||fi|S)a+ Examples: ```python >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> from transformers import AutoProcessor, CLIPSegVisionModel >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegVisionModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```rirdrgrlrrms r-rkzCLIPSegVisionModel.forwards8w|-EPPPr,rn) r%r&r'rQrRrGr r rCr rkrprqs@r-rrsY15Q''$.Q#'+Q+, Q + + QQr,rc peZdZfdZ ddej dedeede e zffd Z ddeffd Z xZ S) rc "t|di|S)a Examples: ```python >>> import torch >>> from transformers import AutoTokenizer, CLIPSegModel >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> with torch.inference_mode(): ... text_features = model.get_text_features(**inputs) ```r+)rjget_text_featuresrs r-rzCLIPSegModel.get_text_featuressw(8<88r,rgrlrrZc &t|||fi|S)a Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegModel >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```)rjget_image_featuresrs r-rzCLIPSegModel.get_image_featuress2w),8P[TZ[[r,c (t|dd|i|y)a return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegModel >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> inputs = processor( ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True ... ) >>> with torch.inference_mode(): ... outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities ```rlNr+ri)rdrlrrms r-rkzCLIPSegModel.forwards6 Z1IZ\Zr,rn)r%r&r'rrQrRrGr r rCr rrkrprqs@r-rrsc9(*.\''\#'\+, \ + + \6[[[r,rzn CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation. ) custom_introceZdZUeed<deffd Z ddedzdejdzdejdzdejdzdejdzd ejf d Z e e ddejdzd ejdzdejdzd ejdzdejdzdejdzd ejdzdedeed eezfdZxZS)CLIPSegForImageSegmentationrct||t||_|j|_t ||_|jyr])rjrrclipr5rdecoderr)rdrrms r-rz$CLIPSegForImageSegmentation.__init__s@   ( $33%f-  r,Nr input_idsr|rconditional_pixel_valuesrZc|`t||k7r tdtj5|jj |||j }ddd|S|]t||k7r tdtj5|jj|j }ddd|Std#1swYSxYw#1swYSxYw)Nz@Make sure to pass as many prompt texts as there are query images)r|rzAMake sure to pass as many prompt images as there are query imagesz[Invalid conditional, should be either provided as `input_ids` or `conditional_pixel_values`)r ValueErrorrQrrr pooler_outputr)rdrrr|rrrVs r-get_conditional_embeddingsz6CLIPSegForImageSegmentation.get_conditional_embeddings s  9~+ !cdd )-)D)Dn<*E*-' &%& 1+, : !dee n)-)E)EF^)_)m)m& n&% m  &% n&%s)C &C CC$rgrVlabelsrlrc  ^tj5d| d<|jjd ||d| } | j} | j } |j D cgc] } | | dz }} t| j| j| j | j} ddd|$|j|jd||||}n[|jd|jdk7r td |jd|jjk7r td |j|fi| }|j }d}|8|j#|j$}t'j(}|||}t+|||  | Scc} w#1swYxYw) a~ conditional_pixel_values (`torch.FloatTensor`, *optional*): The pixel values of the conditional images. conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*): The conditional embeddings for the query images. If provided, the model will use this instead of computing the embeddings from the conditional_pixel_values. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegForImageSegmentation >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> texts = ["a cat", "a remote", "a blanket"] >>> inputs = processor(text=texts, images=[image] * len(texts), padding=True, return_tensors="pt") >>> with torch.inference_mode(): ... outputs = model(**inputs) >>> logits = outputs.logits >>> print(logits.shape) torch.Size([3, 352, 352]) ```Toutput_hidden_states)rgrlr)last_hidden_staterrOrPNr)rrr|rrzWMake sure to pass as many conditional embeddings as there are query images in the batchzcMake sure that the feature dimension of the conditional embeddings matches `config.projection_dim`.)rUrNrVrWrXrYr+)rQrrrrrOr5r rrPrrrrr*rrNtodevicerBCEWithLogitsLossrT)rdrrgrrVr|rrrlrvision_outputsrWrOrrdecoder_outputsrNrUloss_fns r-rkz#CLIPSegForImageSegmentation.forward)sb]]_ -1F) *9TYY99))AN +88M*88M9=9L9LMA=Q/MKM8"0"B"B,::,::)44 N , " )%)%D%D'--a0#-))A &E& "&++A.,2D2DQ2GG m&++A.$++2L2LL 0 '$,,  "   !''  YYv}}-F**,G66*D-#9' .*   [N  sA F# F/9F#F##F,)NNNNN)NNNNNNNT)r%r&r'r2rDrrBrQrorRrrr LongTensorrGr r rCrIrkrprqs@r-rrs }"&)-.2,08< &$J&<<$&& t+ & llT) & #(,,"5 &   &:/315=A;?.204*.)-n $$t+n ''$.n #("3"3d": n !& 1 1D 8 n  t+ n &&-n   4'n #'n +,n   n n r,r)r2r#r0rrrrr)Ar(rr dataclassesrtypingrrQhuggingface_hub.dataclassesrrrrmodeling_outputsr modeling_utilsr processing_utilsr utilsr r r utils.genericrrutils.output_capturingrclip.configuration_cliprrrclip.modeling_cliprrrrrrrrrrr r#r0r2rIrMrTrfrsrurwryr{rrrrrrr__all__r+r,r-rs ! .&:-&DDI5RR    /0&&1&(/0&*&1&(/0+5J+51+5\ J  <;< <" _[_ _6@2@  .  }    * *>(0((V [ U3+U3p/}/(QQ@H[9H[V X "8X  X v r,