
    ^j                        d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dlm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ee G d de                    Z)ee G d de                    Z*ee G d de                    Z+ G d de	jX                        Z- G d de	jX                        Z.	 dHde	jX                  dej^                  dej^                  d ej^                  d!ej^                  dz  d"e0d#e0d$ee   fd%Z1 G d& d'e	jX                        Z2 G d( d)e	jX                        Z3 G d* d+e      Z4 G d, d-e      Z5e G d. d/e             Z6 G d0 d1e	jX                        Z7 G d2 d3e6      Z8 ed45       G d6 d7e6             Z9 ed85       G d9 d:e6             Z:d;ej^                  d<ej^                  fd=Z;d>ej^                  d<ej^                  fd?Z<d@ej^                  d<ej^                  fdAZ=e G dB dCe6             Z> edD5       G dE dFe6             Z?g dGZ@y)I    N)Callable)	dataclass)Any)nn   )initialization)ACT2FN)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstring	torch_int)can_return_tuplemerge_with_config_defaults)capture_outputs   )CLIPSegConfigCLIPSegTextConfigCLIPSegVisionConfigc                      e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   dZeed<   dZeed	<   d
ee   fdZy)CLIPSegOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegTextModel`].
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPSegVisionModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`CLIPSegTextModel`].
    vision_model_output (`BaseModelOutputWithPooling`):
        The output of the [`CLIPSegVisionModel`].
    Nlosslogits_per_imagelogits_per_texttext_embedsimage_embedstext_model_outputvision_model_outputreturnc                 B    t        d | j                         D              S )Nc              3   `   K   | ]&  }t        |t              r|j                         n| ( y wN
isinstancer   to_tuple.0vs     w/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/clipseg/modeling_clipseg.py	<genexpr>z)CLIPSegOutput.to_tuple.<locals>.<genexpr>J   $     ^1Z;%?QZZ\QF^   ,.tuplevaluesselfs    r/   r+   zCLIPSegOutput.to_tupleI       ^PTP[P[P]^^^    )__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   r    r!   r"   r#   r   r$   r4   r   r+    r9   r/   r   r   +   s    & &*D%

d
")15e''$.504OU&&-4,0K""T)0-1L%##d*148186:3:_%* _r9   r   c                       e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   dZ
e	ej                  df   dz  ed<   y)CLIPSegDecoderOutputa  
    logits (`torch.FloatTensor` of shape `(batch_size, height, width)`):
        Classification scores for each pixel.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*,):
        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        Rreturned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`
    attentions (`tuple(torch.FloatTensor)`, *optional*):
        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads. Returned when `output_attentions=True` is passed or when `config.output_attentions=True`
    Nlogits.hidden_states
attentions)r:   r;   r<   r=   rD   r>   r?   r@   rE   r4   rF   rA   r9   r/   rC   rC   M   s\    	 (,FE$+:>M5**C/047>7;Je'',-4;r9   rC   c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeed<   dZeed<   d	ee   fd
Zy)CLIPSegImageSegmentationOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Binary cross entropy loss for segmentation.
    logits (`torch.FloatTensor` of shape `(batch_size, height, width)`):
        Classification scores for each pixel.
    conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, projection_dim)`):
        Conditional embeddings used for segmentation.
    pooled_output (`torch.FloatTensor` of shape `(batch_size, embed_dim)`):
        Pooled output of the [`CLIPSegVisionModel`].
    vision_model_output (`BaseModelOutputWithPooling`):
        The output of the [`CLIPSegVisionModel`].
    decoder_output (`CLIPSegDecoderOutput`):
        The output of the [`CLIPSegDecoder`].
    Nr   rD   conditional_embeddingspooled_outputr$   decoder_outputr%   c                 B    t        d | j                         D              S )Nc              3   `   K   | ]&  }t        |t              r|j                         n| ( y wr(   r)   r,   s     r/   r0   z:CLIPSegImageSegmentationOutput.to_tuple.<locals>.<genexpr>z   r1   r2   r3   r6   s    r/   r+   z'CLIPSegImageSegmentationOutput.to_tupley   r8   r9   )r:   r;   r<   r=   r   r>   r?   r@   rD   rI   rJ   r$   r   rK   rC   r4   r   r+   rA   r9   r/   rH   rH   `   s     &*D%

d
")'+FE$+7;E--4;.2M5$$t+26:3:+/N(/_%* _r9   rH   c                        e Zd Zdef fdZdej                  dededej                  fdZd
dej                  dej                  fd	Z
 xZS )CLIPSegVisionEmbeddingsconfigc                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                              | _        t        j                  |j                  | j                  | j                  | j                  d      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                        | _        | j'                  dt        j(                  | j                         j+                  d      d       y )NF)in_channelsout_channelskernel_sizestridebias   r   position_idsr   
persistent)super__init__rP   hidden_size	embed_dim
image_size
patch_sizer   	Parameterr>   randnclass_embeddingConv2dnum_channelspatch_embeddingnum_patchesnum_positions	Embeddingposition_embeddingregister_bufferarangeexpandr7   rP   	__class__s     r/   r^   z CLIPSegVisionEmbeddings.__init__~   s	   ++ ++ ++!||EKK,GH!yy++?? 
 !OOt>1D!--1"$,,t/A/A4>>"R^U\\$:L:L-M-T-TU\-]jopr9   
embeddingsheightwidthr%   c                    |j                   d   dz
  }| j                  j                  j                  d      }|j                   d   dz
  }t        j
                  j                         s%||k(  r ||k(  r| j                  | j                        S |ddddf   }|ddddf   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz        }|j                  d|||	      }|j                  dddd      }t        j                  j                  ||
|fdd	
      }|j                  dddd      j                  dd|	      }t	        j                   ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   NrZ         ?r   rW   bicubicF)sizemodealign_cornersdim)shaperl   weight	unsqueezer>   jit
is_tracingrX   rb   r   reshapepermuter   
functionalinterpolateviewcat)r7   rr   rs   rt   ri   rl   rj   class_pos_embedpatch_pos_embedr|   
new_height	new_widthsqrt_num_positionss                r/   interpolate_pos_encodingz0CLIPSegVisionEmbeddings.interpolate_pos_encoding   sv    !&&q)A-!44;;EEaH*003a7 yy##%+*F6UZ?**4+<+<==,QU3,QU3r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr9   pixel_valuesc                 `   |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d| d| d| j                   d| j                   d	      | j                  j                  j
                  }| j                  |j                  |            }|j                  d      j                  dd      }| j                  j                  |dd      }	t        j                  |	|gd	      }
|r|
| j                  |
||      z   }
|
S |
| j                  | j                        z   }
|
S )
NzInput image size (*z) doesn't match model (z).)dtyperW   r   rZ   r{   )r}   ra   
ValueErrorrh   r~   r   toflatten	transposere   ro   r>   r   r   rl   rX   )r7   r   r   
batch_size_rs   rt   target_dtypepatch_embedsclass_embedsrr   s              r/   forwardzCLIPSegVisionEmbeddings.forward   s6   '3'9'9$
Avu'Vt-F%SWSbSbJb$VHAeW4KDOOK\\]^b^m^m]nnpq  ++2288++LOO,O,OP#++A.88A>++22:q"EYYl;C
##d&C&CJPVX]&^^J  $d&=&=d>O>O&PPJr9   T)r:   r;   r<   r   r^   r>   Tensorintr   r?   r   __classcell__rq   s   @r/   rO   rO   }   se    q2 q,'D5<< 'D 'DUX 'D]b]i]i 'DRE$5$5 Y^YeYe r9   rO   c            	            e Zd Zdef fdZ	 	 	 d	dej                  dz  dej                  dz  dej                  dz  dej                  fdZ	 xZ
S )
CLIPSegTextEmbeddingsrP   c                 N   t         |           |j                  }t        j                  |j
                  |      | _        t        j                  |j                  |      | _        | j                  dt        j                  |j                        j                  d      d       y )NrX   rY   Fr[   )r]   r^   r_   r   rk   
vocab_sizetoken_embeddingmax_position_embeddingsrl   rm   r>   rn   ro   r7   rP   r`   rq   s      r/   r^   zCLIPSegTextEmbeddings.__init__   s    &&	!||F,=,=yI"$,,v/M/My"Y 	ELL)G)GHOOPWXej 	 	
r9   N	input_idsrX   inputs_embedsr%   c                 8   ||j                   d   n|j                   d   }| j                  j                  j                   d   }||kD  rt        d| d|       || j                  d d d |f   }|| j                  |      }| j                  |      }||z   }|S )NrZ   r   zRSequence length must be less than max_position_embeddings (got `sequence length`: z and max_position_embeddings: )r}   rl   r~   r   rX   r   )r7   r   rX   r   
seq_lengthmax_position_embeddingposition_embeddingsrr   s           r/   r   zCLIPSegTextEmbeddings.forward   s     -6,AY__R(}GZGZ[]G^
!%!8!8!?!?!E!Ea!H..d,<=S<TV 
 ,,Q^<L  00;M"55lC"%88
r9   NNN)r:   r;   r<   r   r^   r>   
LongTensorr?   r   r   r   r   s   @r/   r   r      sk    

0 

 .20426	##d* &&- ((4/	
 
r9   r   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )NrZ   r   )r|   r   )ptrainingr   rW   )r>   matmulr   r   r   softmaxfloat32r   r   r   r   
contiguous)
r   r   r   r   r   r   r   r   attn_weightsattn_outputs
             r/   eager_attention_forwardr      s     <<s}}R'<=GL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r9   c                        e Zd ZdZdeez  f fdZ	 d
dej                  dej                  dz  de	e
   deej                  ej                  dz  f   fd	Z xZS )CLIPSegAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrP   c                    t         |           || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  dz  | _        |j                  | _
        d| _        t        j                  | j                  | j                        | _        t        j                  | j                  | j                        | _        t        j                  | j                  | j                        | _        t        j                  | j                  | j                        | _        y )N      F)r]   r^   rP   r_   r`   num_attention_heads	num_headshead_dimscaleattention_dropoutr   	is_causalr   Lineark_projv_projq_projout_projrp   s     r/   r^   zCLIPSegAttention.__init__  s    ++33$..8]]D(
//ii?ii?ii?		$..$..Ar9   NrE   r   r   r%   c                    |j                   dd }g |d| j                  }| j                  |      }| j                  |      }| j	                  |      }|j                  |      j                  dd      }|j                  |      j                  dd      }|j                  |      j                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  | j                  sdn| j                  d|\  }
} |
j                  g |d j!                         }
| j#                  |
      }
|
|fS )z#Input shape: Batch x Time x ChannelNrZ   r   rW           )r   r   )r}   r   r   r   r   r   r   r   get_interfacerP   _attn_implementationr   r   r   r   r   r   r   )r7   rE   r   r   input_shapehidden_shapequerieskeysr5   attention_interfacer   r   s               r/   r   zCLIPSegAttention.forward  sO    $))#2.88b8$--8++m,{{=)]+,,|,66q!<yy&00A6\*44Q:(?(M(MKK,,.E)
 %8	%
 JJ#}}C$,,	%
 	%
!\ *k));;;;FFHmmK0L((r9   r(   )r:   r;   r<   r=   r   r   r^   r>   r   r   r   r4   r   r   r   s   @r/   r   r     su    GB25FF B$ /3%)||%) t+%) +,	%)
 
u||U\\D00	1%)r9   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )
CLIPSegMLPc                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y r(   )r]   r^   rP   r	   
hidden_actactivation_fnr   r   r_   intermediate_sizefc1fc2rp   s     r/   r^   zCLIPSegMLP.__init__H  sd    #F$5$5699V//1I1IJ99V55v7I7IJr9   rE   r%   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r(   )r   r   r   )r7   rE   s     r/   r   zCLIPSegMLP.forwardO  s4    /**=9/r9   )r:   r;   r<   r^   r>   r   r   r   r   s   @r/   r   r   G  s$    KU\\ ell r9   r   c                        e Zd Zdeez  f fdZdej                  dej                  dee	   dej                  fdZ xZS )CLIPSegEncoderLayerrP   c                 D   t         |           |j                  | _        t	        |      | _        t        j                  | j                  |j                        | _	        t        |      | _        t        j                  | j                  |j                        | _        y N)epsr]   r^   r_   r`   r   	self_attnr   	LayerNormlayer_norm_epslayer_norm1r   mlplayer_norm2rp   s     r/   r^   zCLIPSegEncoderLayer.__init__W  m    ++)&1<<F<Q<QRf%<<F<Q<QRr9   rE   r   r   r%   c                     |}| j                  |      } | j                  d||d|\  }}||z   }|}| j                  |      }| j                  |      }||z   }|S N)rE   r   rA   )r   r   r   r   r7   rE   r   r   residualr   s         r/   r   zCLIPSegEncoderLayer.forward_  s     !((7)4>> 
')
 
q
 !=0 ((7/ =0r9   )r:   r;   r<   r   r   r^   r>   r   r   r   r?   r   r   r   s   @r/   r   r   V  sW    S25FF S||  +,	
 
		r9   r   c                        e Zd ZdZdeez  f fdZdej                  dej                  de	ej                     fdZ xZS )CLIPSegDecoderLayerz
    CLIPSeg decoder layer, which is identical to `CLIPSegEncoderLayer`, except that normalization is applied after
    self-attention/MLP, rather than before.
    rP   c                 D   t         |           |j                  | _        t	        |      | _        t        j                  | j                  |j                        | _	        t        |      | _        t        j                  | j                  |j                        | _        y r   r   rp   s     r/   r^   zCLIPSegDecoderLayer.__init__}  r   r9   rE   r   r%   c                     |} | j                   d||d|\  }}||z   }| j                  |      }|}| j                  |      }||z   }| j                  |      }|S r   )r   r   r   r   r   s         r/   r   zCLIPSegDecoderLayer.forward  s     !)4>> 
')
 
q !=0((7 / =0((7r9   )r:   r;   r<   r=   r   r   r^   r>   r   r4   r?   r   r   r   s   @r/   r   r   w  sQ    
S25FF S|| 
 
u  	!r9   r   c                        e Zd ZU eed<   dZdZg dZdZdZ	dZ
dZdZeegedZ ej$                          fd       Z xZS )CLIPSegPreTrainedModelrP   clipseg)imagetext)r   r   rO   T)rE   rF   c                    t         |   |       | j                  j                  }t	        |t
              rt        j                  |j                  j                  d|dz         t        j                  |j                  j                  d|dz         t        j                  |j                  t        j                  |j                  j                  d         j!                  d             y	t	        |t"              rt        j                  |j$                  d|j&                  dz  |z         t        j                  |j(                  j                  |j                  j*                  |z         t        j                  |j                  j                  |j                  j*                  |z         t        j                  |j                  t        j                  |j,                        j!                  d             y	t	        |t.              r|j&                  dz  d|j                  j0                  z  dz  z  |z  }|j&                  dz  |z  }t        j                  |j2                  j                  |       t        j                  |j4                  j                  |       t        j                  |j6                  j                  |       t        j                  |j8                  j                  |       y	t	        |t:              r|j                  j<                  dz  d|j                  j0                  z  dz  z  |z  }d|j                  j<                  z  dz  |z  }t        j                  |j>                  j                  |       t        j                  |j@                  j                  |       y	t	        |tB              rwt        j                  |jD                  j                  |jF                  dz  |z         t        j                  |jH                  j                  |jJ                  dz  |z         y	y	)
zInitialize the weightsr   g{Gz?)meanstdrZ   rY   r   )r   rW   N)&r]   _init_weightsrP   initializer_factorr*   r   initnormal_r   r~   rl   copy_rX   r>   rn   r}   ro   rO   re   r`   rh   initializer_rangerj   r   num_hidden_layersr   r   r   r   r   r_   r   r   CLIPSegModeltext_projectiontext_embed_dimvisual_projectionvision_embed_dim)r7   r   factorin_proj_stdout_proj_stdfc_stdrq   s         r/   r  z$CLIPSegPreTrainedModel._init_weights  s    	f%//f34LL//66SftmTLL2299RVWJJv**ELL9L9L9R9RSU9V,W,^,^_f,gh 78LL//cv?O?OQU?UX^?^_LL//66FMM<[<[^d<deLL2299v}}?^?^ag?ghJJv**ELL9M9M,N,U,UV],^_ 01!++T1q6==;Z;Z7Z_c6cdgmmK",,d2f<LLL--;?LL--;?LL--;?LL//\B
+!==44d:FMMDcDc@chl?lmpvvK&--333<vEFLL**7LL**<-LL&&--))4/&8 LL((//++T1F: .r9   )r:   r;   r<   r   r@   base_model_prefixinput_modalities_no_split_modulessupports_gradient_checkpointing_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendr   r   r   _can_record_outputsr>   no_gradr  r   r   s   @r/   r   r     sh    !(c&*#N"&-/BC&
 U]]_! !r9   r   c                   `     e Zd ZdZdef fdZ	 d	dej                  dz  dee	   de
fdZ xZS )
CLIPSegEncoderz
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`CLIPSegEncoderLayer`].

    Args:
        config: CLIPSegConfig
    rP   c                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w )NF)
r]   r^   rP   r   
ModuleListranger  r   layersgradient_checkpointing)r7   rP   r   rq   s      r/   r^   zCLIPSegEncoder.__init__  sP    mm%PVPhPhJi$jQ%8%@$jk&+# %ks   A#Nr   r   r%   c                 T    |}| j                   D ]  } |||fi |} t        |      S )N)last_hidden_state)r   r   )r7   r   r   r   rE   encoder_layers         r/   r   zCLIPSegEncoder.forward  sH     &![[ 	M) M	 +
 	
r9   r(   )r:   r;   r<   r=   r   r^   r>   r   r   r   r   r   r   r   s   @r/   r  r    sK    ,} , /3
 t+
 +,	

 

r9   r  c                        e Zd Zdef fdZeeedee	j                     de	j                  dee   defd                     Z xZS )CLIPSegDecoderrP   c                 &   t         |   |       |j                  | _        t        j                  |j
                  |j                        | _        t        j                  |j
                  |j                        | _        |j                  r|j                  j                  dz  |j                  j                  dz  f}t        j                  t        j                  |j                  |j                  dd      t        j                         t        j                  |j                  |j                  dz  |d   |d         t        j                         t        j                  |j                  dz  d|d   |d               | _        nPt        j                  |j                  d|j                  j                  |j                  j                        | _        t#        |j$                        }t        j&                  t)        |      D cg c]6  }t        j                  |j                  j*                  |j                        8 c}      | _        t/        j0                  |j                        }|j                  |_        |j2                  |_        |j6                  |_        d	|_        t        j&                  t)        t#        |j$                              D cg c]  }t=        |       c}      | _        | jA                          y c c}w c c}w )
N   r   r   )rT   paddingrW   r   )rT   rU   )rU   relu)!r]   r^   conditional_layerr   r   projection_dim
reduce_dimfilm_mulfilm_add"use_complex_transposed_convolutionvision_configrb   
Sequentialrf   ReLUConvTranspose2dtransposed_convolutionlenextract_layersr  r  r_   reducescopydeepcopydecoder_num_attention_headsr   decoder_intermediate_sizer   r   r   r   	post_init)r7   rP   transposed_kernelsdepthr   decoder_configrq   s         r/   r^   zCLIPSegDecoder.__init__  s[    !'!9!9		&"7"79J9JK		&"7"79J9JK44"("6"6"A"AQ"FH\H\HgHgklHl!m*,--		&++V->->AWXY	""%%%%* 21 5-a0	 	""%%*A;Ma;PYklmYn+D' +-*<*<!!1f&:&:&E&EfNbNbNmNm+D' F))*}}UZ[`UabPQRYYv++779J9JKb
 v';';<%+%6%6"-3-O-O*+1+K+K($*!mmRWX[\b\q\qXrRs$tQ%8%H$tu c %us   ;L	LrE   rI   r   r%   c                    |ddd   }d}t        t        || j                  | j                              D ]  \  }\  }}}	| |	|      |z   }n |	|      }|| j                  k(  rJ| j                  |      |j                  ddd      z  | j                  |      z   }|j                  ddd      } ||fddi|} |ddddddf   j                  dd      }t        t        j                  |j                  d               }
|j                  d   }|j                  ||j                  d   |
|
      }| j                  |      j                  d      }t!        |      S )a/  
        conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*):
            The conditional embeddings for the query images. If provided, the model will use this instead of computing
            the embeddings from the conditional_pixel_values.
        NrZ   r   r   rW   r   rD   )	enumeratezipr   r8  r+  r.  r   r/  r   r   mathsqrtr}   r   r5  squeezerC   )r7   rE   rI   r   activationsoutputi
activationlayerreducerx   r   rD   s                r/   r   zCLIPSegDecoder.forward#  sk    $DbD).7KVZVbVb8c.d 	B*A*
E6!
+f4
+D***'=>PQSTVWAXX[_[h[h*\   1a06A$A&AF	B 12q!++Aq1499V\\!_-.+11!4
Za$E,,V4<<Q?#622r9   )r:   r;   r<   r   r^   r   r   r   r4   r>   r   r   r   rC   r   r   r   s   @r/   r&  r&    sm    *} *X  %3U\\*%3 !&%3 +,	%3
 
%3    %3r9   r&  zL
    The text model from CLIPSEG without any head or projection on top.
    )custom_introc                        e Zd ZU eed<   dZdZdef fdZe e	d      e
	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dee   deez  f
d                     Z xZS )CLIPSegTextModelrP   )r   r   c                 
   t         |   |       |j                  }t        |      | _        t        |      | _        t        j                  ||j                        | _
        |j                  | _        | j                          y r   )r]   r^   r_   r   rr   r  encoderr   r   r   final_layer_normeos_token_idr=  r   s      r/   r^   zCLIPSegTextModel.__init__X  sf     &&	/7%f- "YF<Q<Q R #//r9   Ftie_last_hidden_statesNr   r   rX   r   r%   c                 h   |t        d      |j                         }|j                  d|d         }| j                  ||      }t	        | j
                  ||d      }|j                  dd        | j                  d||dd|}|j                  }| j                  |      }| j                  d	k(  rm|t        j                  |j                  d
   |j                        |j                  t        j                   |j                        j#                  d      f   }	n|t        j                  |j                  d
   |j                        |j                  t        j                   |j                        | j                  k(  j!                         j#                  d      f   }	t%        ||	      S )a;  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, CLIPSegTextModel

        >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegTextModel.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```NzYou have to specify input_idsrZ   )r   rX   )rP   r   r   past_key_valuesr   T)r   r   r   rW   r   device)r   rZ  r{   r#  pooler_outputrA   )r   rx   r   rr   r
   rP   poprR  r#  rS  rT  r>   rn   r}   rZ  r   r   argmaxr   )
r7   r   r   rX   r   r   rE   encoder_outputsr#  rJ   s
             r/   r   zCLIPSegTextModel.forwardc  s   2 <==nn&NN2{27	),W+;;') 	
 	

;%+74<< ,
'),
 	,
 ,== 112CD! ..44Q7@Q@X@XY5995F5M5MNUUZ\U]_M ..44Q7@Q@X@XY EII6G6N6NOSWSdSddB!M */'
 	
r9   r   )r:   r;   r<   r   r@   r  _input_embed_layerr^   r   r   r   r>   r   r   r   r4   r   r   r   r   s   @r/   rP  rP  N  s      *	0 	  E2 *..2,0	I
<<$&I
 t+I
 llT)	I

 +,I
 
+	+I
  3  I
r9   rP  zN
    The vision model from CLIPSEG without any head or projection on top.
    c                        e Zd ZU eed<   dZdZdZdef fdZe	 e
d      e	 ddej                  dz  d	edz  d
ee   deez  fd                     Z xZS )CLIPSegVisionModelrP   r   )r   rh   c                 4   t         |   |       |j                  }t        |      | _        t        j                  ||j                        | _        t        |      | _
        t        j                  ||j                        | _        | j                          y r   )r]   r^   r_   rO   rr   r   r   r   pre_layrnormr  rR  post_layernormr=  r   s      r/   r^   zCLIPSegVisionModel.__init__  so     &&	1&9LL8M8MN%f- ll9&:O:OPr9   FrU  Nr   r   r%   c                     | j                  ||      }| j                  |      } | j                  dd|i|}|j                  }|dddddf   }| j	                  |      }t        ||      S )a+  
        Examples:

        ```python
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> from transformers import AutoProcessor, CLIPSegVisionModel

        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegVisionModel.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```)r   r   Nr   r[  rA   )rr   rd  rR  r#  re  r   )r7   r   r   r   rE   r_  r#  rJ   s           r/   r   zCLIPSegVisionModel.forward  s    > Ogh))-8+74<< ,
',
,

 ,==)!Q'2++M:)/'
 	
r9   r   )r:   r;   r<   r   r@   main_input_namer  r`  r^   r   r   r   r>   r?   boolr   r   r4   r   r   r   r   s   @r/   rb  rb    s      $O!*2   E2 15+
''$.+
 #'++
 +,	+

 
+	++
  3  +
r9   rb  rD   r%   c                     t         j                  j                  | t        j                  t        |       | j                              S )NrY  )r   r   cross_entropyr>   rn   r6  rZ  rB  s    r/   contrastive_lossrk    s/    ==&&vu||CKPVP]P]/^__r9   
similarityc                 R    t        |       }t        | j                        }||z   dz  S )Ng       @)rk  T)rl  caption_loss
image_losss      r/   image_text_contrastive_lossrq    s*    #J/L!*,,/J:%,,r9   tensorc                     t        j                  | d      }t        j                  |dd      }t        j                  |d      }|S )z
    This method is equivalent to tensor.norm(p=2, dim=-1, keepdim=True) and used to make
    model `executorch` exportable. See issue https://github.com/pytorch/executorch/issues/3566
    rW   rZ   T)r|   keepdimrv   )r>   powsum)rr  square_tensor
sum_tensornormed_tensors       r/   _get_vector_normrz    s<    
 IIfa(M=b$?JIIj#.Mr9   c                       e Zd Zdef fdZee	 	 ddej                  dej                  dz  dej                  dz  de	e
   deez  f
d	              Zee	 dd
ej                  dede	e
   deez  fd              Zee	 	 	 	 	 	 ddej"                  dz  d
ej                  dz  dej                  dz  dej"                  dz  dedz  dede	e
   defd              Z xZS )r  rP   c                    t         |   |       |j                  }|j                  }|j                  | _        |j
                  | _        |j
                  | _        t        j                  |      | _
        t        j                  |      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j"                  t%        j&                  | j(                  j*                              | _        | j/                          y )NF)rV   )r]   r^   text_configr1  r,  r_   r
  r  rP  _from_config
text_modelrb  vision_modelr   r   r  r	  rc   r>   rr  rP   logit_scale_init_valuelogit_scaler=  )r7   rP   r}  r1  rq   s       r/   r^   zCLIPSegModel.__init__  s     ((,,$33)55 - 9 9*77D.;;MJ!#4+@+@$BUBU\a!b!yy)<)<d>Q>QX]^<<T[[5W5W(XY 	r9   Nr   r   rX   r   r%   c                 x     | j                   d|||dd|}|j                  }| j                  |      |_        |S )a  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoTokenizer, CLIPSegModel

        >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     text_features = model.get_text_features(**inputs)
        ```T)r   r   rX   return_dictrA   )r  r\  r	  )r7   r   r   rX   r   text_outputsrJ   s          r/   get_text_featureszCLIPSegModel.get_text_features%  sV    . 4C4?? 4
)%	4

 4
 %22%)%9%9-%H"r9   r   r   c                 v     | j                   d||dd|}|j                  }| j                  |      |_        |S )a  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, CLIPSegModel
        >>> from transformers.image_utils import load_image

        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> with torch.inference_mode():
        ...     image_features = model.get_image_features(**inputs)
        ```T)r   r   r  rA   )r  r\  r  )r7   r   r   r   vision_outputsrJ   s         r/   get_image_featureszCLIPSegModel.get_image_featuresH  sU    6 6GT5F5F 6
%%=6
 	6
 '44'+'='=m'L$r9   return_lossc           	          | j                   d||d|} | j                  d|||d|}	|j                  }
|	j                  }|
t        |
      z  }
|t        |      z  }t	        j
                  ||
j                         j                  |j                              }|| j                  j                         j                  |j                        z  }|j                         }d}|rt        |      }t        |||||
|	|      S )a  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, CLIPSegModel
        >>> from transformers.image_utils import load_image

        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(
        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
        ... )

        >>> with torch.inference_mode():
        ...     outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```r   r   )r   r   rX   N)r   r   r    r!   r"   r#   r$   rA   )r  r  r\  rz  r>   r   tr   rZ  r  exprq  r   )r7   r   r   r   rX   r  r   r   r  r  r"   r!   r    r   r   s                  r/   r   zCLIPSegModel.forwardn  s+   L 6MT5L5L 6
%%=6
 6
 4J43I3I 4
)%4
 	4
 &33"00 $&6|&DD!$4[$AA  ,,{LNN4D4G4GHZHZ4[\)D,<,<,@,@,B,E,EkFXFX,YY*,,..?D-+#%* .
 	
r9   )NNr   )NNNNNT)r:   r;   r<   r   r^   r   r   r>   r   r   r   r4   r   r  r?   rh  r  r   r   r   r   r   s   @r/   r  r    s   } (  /3,0	<< t+ llT)	
 +, 
+	+  B  *."''" #'" +,	"
 
+	+"  "H  .215.204#')-J
##d*J
 ''$.J
 t+	J

 &&-J
 D[J
 #'J
 +,J
 
J
  J
r9   r  zn
    CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation.
    c                       e Zd ZU eed<   def fdZ	 	 	 	 	 ddedz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  fd
Z
ee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedee   d	eez  fd              Z xZS )CLIPSegForImageSegmentationrP   c                     t         |   |       t        |      | _        |j                  | _        t        |      | _        | j                          y r(   )r]   r^   r  clipr7  r&  decoderr=  rp   s     r/   r^   z$CLIPSegForImageSegmentation.__init__  s@      (	$33%f-r9   Nr   r   r   rX   conditional_pixel_valuesr%   c                    |`t        |      |k7  rt        d      t        j                         5  | j                  j                  |||      j                  }d d d        |S |]t        |      |k7  rt        d      t        j                         5  | j                  j                  |      j                  }d d d        |S t        d      # 1 sw Y   S xY w# 1 sw Y   S xY w)Nz@Make sure to pass as many prompt texts as there are query images)r   rX   zAMake sure to pass as many prompt images as there are query imagesz[Invalid conditional, should be either provided as `input_ids` or `conditional_pixel_values`)r6  r   r>   r  r  r  r\  r  )r7   r   r   r   rX   r  rI   s          r/   get_conditional_embeddingsz6CLIPSegForImageSegmentation.get_conditional_embeddings  s      9~+ !cdd  )-)D)Dn< *E *- '  &% &1+,
: !dee n)-)E)EF^)_)m)m&n &%	 m   &%n &%s   )C&CCC$r   rI   labelsr   r   c	                 ^   t        j                         5  d|	d<    | j                  j                  d||d|	}
|
j                  }|
j
                  }| j                  D cg c]
  }||dz       }}t        |
j                  |
j                  |
j
                  |
j                        }
ddd       |$| j                  |j                  d   ||||      }n[|j                  d   |j                  d   k7  rt        d	      |j                  d   | j                  j                  k7  rt        d
       | j                  |fi |	}|j                   }d}|8|j#                  |j$                        }t'        j(                         } |||      }t+        |||
|      S c c}w # 1 sw Y   xY w)a~  
        conditional_pixel_values (`torch.FloatTensor`, *optional*):
            The pixel values of the conditional images.
        conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*):
            The conditional embeddings for the query images. If provided, the model will use this instead of computing
            the embeddings from the conditional_pixel_values.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, CLIPSegForImageSegmentation
        >>> from transformers.image_utils import load_image

        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
        >>> model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> texts = ["a cat", "a remote", "a blanket"]
        >>> inputs = processor(text=texts, images=[image] * len(texts), padding=True, return_tensors="pt")

        >>> with torch.inference_mode():
        ...     outputs = model(**inputs)

        >>> logits = outputs.logits
        >>> print(logits.shape)
        torch.Size([3, 352, 352])
        ```Toutput_hidden_statesr  r   )r#  r\  rE   rF   Nr   )r   r   r   rX   r  zWMake sure to pass as many conditional embeddings as there are query images in the batchzcMake sure that the feature dimension of the conditional embeddings matches `config.projection_dim`.)r   rD   rI   rJ   r$   rK   rA   )r>   r  r  r  r\  rE   r7  r   r#  rF   r  r}   r   rP   r,  r  rD   r   rZ  r   BCEWithLogitsLossrH   )r7   r   r   r  rI   r   rX   r  r   r   r  rJ   rE   rJ  rH  decoder_outputsrD   r   loss_fns                      r/   r   z#CLIPSegForImageSegmentation.forward  s   b ]]_ 	-1F)*9TYY99 ))A N
 +88M*88M9=9L9LMA=Q/MKM 8"0"B"B,::,::)44	N	, ")%)%D%D'--a0#-))A &E &" &++A.,2D2DQ2GG m  &++A.$++2L2LL 0  '$,,"
 

 !''YYv}}-F**,G66*D-#9' .*
 	
[ N	 	s   AF# F/9F#F##F,)NNNNN)NNNNNNNT)r:   r;   r<   r   r@   r^   r   r>   r   r?   r  r   r   r   rh  r   r   r4   r   r   r   r   s   @r/   r  r    s    }  "&)-.2,08<&$J& <<$&& t+	&
 llT)& #(,,"5& 
		&:  /315=A;?.204*.)-n
$$t+n
 ''$.n
 #("3"3d":	n

 !& 1 1D 8n
 t+n
 &&-n
   4'n
 #'n
 +,n
 
	n
  n
r9   r  )r  r   rP  rb  r  )r   )Ar9  rE  collections.abcr   dataclassesr   typingr   r>   r    r   r  rH  r	   masking_utilsr
   modeling_layersr   modeling_outputsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   configuration_clipsegr   r   r   r   rC   rH   ModulerO   r   r   floatr   r   r   r   r   r   r  r&  rP  rb  rk  rq  rz  r  r  __all__rA   r9   r/   <module>r     s  *   $ !    & ! / 9 K F & O O I 5 X X 
_K _  _@ 
<; <  <" 
_[ _  _6Pbii Pf%BII %^ %II%<<% 
% <<	%
 LL4'% % % '(%*7)ryy 7)t 4 B$4 $N 2_ 2 2j
RYY 
DU3+ U3p 
\
- \

\
~ 
>
/ >

>
F`U\\ `ell `-ELL -U\\ -U\\ ell  j
) j
 j
Z 
X
"8 X

X
vr9   