Ë
    ÛÓ^j1Œ  ã                   óF  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddlm	Z
 ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'  e"jP                  e)«      Z* G d„ dejV                  «      Z, G d„ dejV                  «      Z- G d„ dejV                  «      Z. G d„ de«      Z/e! G d„ de«      «       Z0e! G d„ de0«      «       Z1 e!d ¬!«       G d"„ d#e0e«      «       Z2 e!d$¬!«       G d%„ d&e0«      «       Z3g d'¢Z4y)(zPyTorch OpenAI ImageGPT model.é    N)ÚAny)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚ SequenceClassifierOutputWithPast)ÚPreTrainedModel)ÚConv1D)Úauto_docstringÚloggingÚtorch_float)Úmaybe_autocasté   )ÚImageGPTConfigc                   óh   ‡ — e Zd Zddee   defˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )ÚImageGPTLayerNormÚhidden_sizeÚepsc                 óŠ   •— t         ‰| �  «        || _        t        j                  t        j                  |«      «      | _        y ©N)ÚsuperÚ__init__r   r   Ú	ParameterÚtorchÚTensorÚweight)Úselfr   r   Ú	__class__s      €úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/imagegpt/modeling_imagegpt.pyr"   zImageGPTLayerNorm.__init__1   s.   ø€ Ü‰ÑÔØˆŒÜ—l‘l¤5§<¡<°Ó#<Ó=ˆ�ó    ÚtensorÚreturnc                 óÀ   — |t        j                  t        j                  t        j                  |«      dd¬«      | j                  z   «      z  }|| j
                  z  }|S )NéÿÿÿÿT)ÚaxisÚkeepdim)r$   ÚsqrtÚmeanÚsquarer   r&   )r'   r+   s     r)   ÚforwardzImageGPTLayerNorm.forward6   sK   € àœ%Ÿ*™*¤U§Z¡Z´·±¸VÓ0DÈ2ÐW[Ô%\Ð_c×_gÑ_gÑ%gÓhÑhˆØ˜$Ÿ+™+Ñ%ˆØˆr*   )gñhãˆµøä>)Ú__name__Ú
__module__Ú__qualname__ÚtupleÚintÚfloatr"   r$   r%   r4   Ú__classcell__©r(   s   @r)   r   r   0   s5   ø„ ñ> E¨#¡Jð >°Uõ >ð
˜eŸl™lð ¨u¯|©|÷ r*   r   c                   óü   ‡ — e Zd Zddedz  dedz  fˆ fd„Zdd„Zdd„Zd„ Zd„ Z		 	 	 	 	 	 dd	e
j                  d
edz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  defd„Zˆ xZS )ÚImageGPTAttentionNÚis_cross_attentionÚ	layer_idxc           	      óž  •— t         ‰| �  «        || _        |j                  }| j	                  dt        j                  t        j                  ||ft
        j                  ¬«      «      j                  dd||«      d¬«       |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | _        | j                  | j                  z  | j                  k7  r&t!        d| j                  › d| j                  › d�«      ‚|j"                  | _        || _        |j&                  | _        || _        |j*                  | _        | j$                  rNt-        d	| j                  z  | j                  «      | _        t-        | j                  | j                  «      | _        n(t-        d
| j                  z  | j                  «      | _        t-        | j                  | j                  «      | _        t5        j6                  |j8                  «      | _        t5        j6                  |j<                  «      | _        y )NÚbias©Údtyper   F)Ú
persistentz=`embed_dim` must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).é   r   ) r!   r"   ÚconfigÚmax_position_embeddingsÚregister_bufferr$   ÚtrilÚonesÚboolÚviewr   Ú	embed_dimÚnum_attention_headsÚ	num_headsÚhead_dimÚ
split_sizeÚ
ValueErrorÚscale_attn_weightsr?   Úscale_attn_by_inverse_layer_idxr@   Úreorder_and_upcast_attnr   Úc_attnÚq_attnÚc_projr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropout)r'   rG   r?   r@   Úmax_positionsr(   s        €r)   r"   zImageGPTAttention.__init__>   sÌ  ø€ Ü‰ÑÔØˆŒØ×6Ñ6ˆØ×ÑØÜ�J‰J”u—z‘z =°-Ð"@ÌÏ
É
ÔSÓT×YÑYØ�1�m ]óð ð 	ô 	
ð  ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØŸ.™.ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØOÐPT×P^ÑP^ÐO_ð `Ø—N‘NÐ# 2ð'óð ð
 #)×";Ñ";ˆÔØ"4ˆÔð 06×/UÑ/UˆÔ,Ø"ˆŒØ'-×'EÑ'EˆÔ$à×"Ò"Ü   T§^¡^Ñ!3°T·^±^ÓDˆDŒKÜ  §¡°·±Ó@ˆD�Kä   T§^¡^Ñ!3°T·^±^ÓDˆDŒKÜ˜TŸ^™^¨T¯^©^Ó<ˆŒäŸJ™J v×'8Ñ'8Ó9ˆÔÜŸZ™Z¨×(:Ñ(:Ó;ˆÕr*   c                 ó6  — t        j                  ||j                  dd«      «      }| j                  r |t	        |j                  d«      dz  «      z  }| j                  r|t        | j                  dz   «      z  }| j                  s¬|j                  d«      |j                  d«      }}| j                  d d …d d …||z
  |…d |…f   }t        j                  |j                  «      j                  }	t        j                  |	|j                  |j                  ¬«      }	t        j                   |||	«      }|�||z   } t#        j$                  d¬«      |«      }|j'                  |j                  «      }| j)                  |«      }t        j                  ||«      }
|
|fS )Nr.   éþÿÿÿç      à?r   ©rD   Údevice©Údim)r$   ÚmatmulÚ	transposerT   r   ÚsizerU   r:   r@   r?   rB   ÚfinforD   Úminr+   rd   Úwherer   ÚSoftmaxÚtyper\   )r'   ÚqueryÚkeyÚvalueÚattention_maskÚattn_weightsÚquery_lengthÚ
key_lengthÚcausal_maskÚ
mask_valueÚattn_outputs              r)   Ú_attnzImageGPTAttention._attnf   sb  € Ü—|‘| E¨3¯=©=¸¸RÓ+@ÓAˆà×"Ò"Ø'¬+°e·j±jÀ³nÈÑ6KÓ*LÑLˆLð ×/Ò/Ø'¬%°·±ÀÑ0BÓ*CÑCˆLà×&Ò&à',§z¡z°"£~°s·x±xÀ³|˜*ˆLØŸ)™)¢A¢q¨*°|Ñ*CÀjÐ*PÐR]ÐS]ÐR]Ð$]Ñ^ˆKÜŸ™ \×%7Ñ%7Ó8×<Ñ<ˆJô Ÿ™ j¸×8JÑ8JÐS_×SfÑSfÔgˆJÜ Ÿ;™; {°LÀ*ÓMˆLàÐ%à'¨.Ñ8ˆLà)”r—z‘z bÔ)¨,Ó7ˆð $×(Ñ(¨¯©Ó5ˆØ×(Ñ(¨Ó6ˆä—l‘l <°Ó7ˆà˜LÐ(Ð(r*   c                 ój  — |j                  «       \  }}}}|j                  «       \  }	}	}
}	t        j                  ||z  ||
t        j                  |j                  ¬«      }d}| j
                  r |t        |j                  d«      «      dz  z  }| j                  r|t        | j                  dz   «      z  }t        |j                  j                  d¬«      5  |j                  d||«      |j                  dd«      j                  d||
«      }}t        j                  ||j                  «       |j                  «       d	|¬
«      }|j                  ||||
«      }d d d «       | j                  s¬|j                  d«      |j                  d«      }}| j                  d d …d d …||z
  |…d |…f   }t        j                   |j"                  «      j$                  }t        j&                  ||j"                  |j                  ¬«      }t        j(                  |||«      }|�||z   } t+        j,                  d¬«      |«      }|j"                  t        j                  k7  rt/        d«      ‚|j                  |j"                  «      }| j1                  |«      }t        j2                  ||«      }||fS # 1 sw Y   �ŒSxY w)Nrc   g      ð?r.   rb   r   F)Úenabledra   r   )ÚbetaÚalphare   zDError with upcasting, attn_weights does not have dtype torch.float32)ri   r$   ÚemptyÚfloat32rd   rT   r:   rU   r@   r   rn   Úreshaperh   Úbaddbmmr?   rB   rj   rD   rk   r+   rl   r   rm   ÚRuntimeErrorr\   rg   )r'   ro   rp   rq   rr   ÚbszrP   Ú	q_seq_lenÚdkÚ_Ú	k_seq_lenrs   Úscale_factorÚqÚkrt   ru   rv   rw   rx   s                       r)   Ú_upcast_and_reordered_attnz,ImageGPTAttention._upcast_and_reordered_attnˆ   sZ  € à(-¯
©
«Ñ%ˆˆY˜	 2Ø ŸX™X›ZÑˆˆ1ˆi˜ô —{‘{ 3¨¡?°I¸yÔPU×P]ÑP]Ðfk×frÑfrÔsˆð ˆØ×"Ò"ØœE %§*¡*¨R£.Ó1°SÑ8Ñ8ˆLà×/Ò/ØœE $§.¡.°1Ñ"4Ó5Ñ5ˆLô ˜EŸL™L×-Ñ-°uÔ=ñ 	VØ—=‘=  Y°Ó3°S·]±]À2ÀrÓ5J×5RÑ5RÐSUÐWYÐ[dÓ5eˆqˆAÜ Ÿ=™=¨°q·w±w³yÀ!Ç'Á'Ã)ÐRSÐ[gÔhˆLØ'×/Ñ/°°YÀ	È9ÓUˆL÷	Vð
 ×&Ò&à',§z¡z°"£~°s·x±xÀ³|˜*ˆLØŸ)™)¢A¢q¨*°|Ñ*CÀjÐ*PÐR]ÐS]ÐR]Ð$]Ñ^ˆKÜŸ™ \×%7Ñ%7Ó8×<Ñ<ˆJô Ÿ™ j¸×8JÑ8JÐS_×SfÑSfÔgˆJÜ Ÿ;™; {°LÀ*ÓMˆLàÐ%à'¨.Ñ8ˆLà)”r—z‘z bÔ)¨,Ó7ˆð ×Ñ¤§¡Ò.ÜÐeÓfÐfØ#×(Ñ(¨¯©Ó5ˆØ×(Ñ(¨Ó6ˆä—l‘l <°Ó7ˆà˜LÐ(Ð(÷;	Vñ 	Vús   ÃBJ(Ê(J2c                 óx   — |j                  «       dd ||fz   } |j                  |Ž }|j                  dddd«      S )zJ
        Splits hidden_size dim into attn_head_size and num_heads
        Nr.   r   rF   r   r   )ri   rM   Úpermute©r'   r+   rP   Úattn_head_sizeÚ	new_shapes        r)   Ú_split_headszImageGPTAttention._split_heads¸   sE   € ð —K‘K“M # 2Ð&¨)°^Ð)DÑDˆ	Ø�—‘˜iÐ(ˆØ�~‰~˜a  A qÓ)Ð)r*   c                 óœ   — |j                  dddd«      j                  «       }|j                  «       dd ||z  fz   }|j                  |«      S )zS
        Merges attn_head_size dim and num_attn_heads dim into hidden_size
        r   rF   r   r   Nra   )r�   Ú
contiguousri   rM   rŽ   s        r)   Ú_merge_headszImageGPTAttention._merge_headsÀ   sO   € ð —‘  1 a¨Ó+×6Ñ6Ó8ˆØ—K‘K“M # 2Ð&¨)°nÑ*DÐ)FÑFˆ	Ø�{‰{˜9Ó%Ð%r*   Úhidden_statesÚ
layer_pastrr   Úencoder_hidden_statesÚencoder_attention_maskÚ	use_cacheÚoutput_attentionsr,   c                 ój  — |d u}	|j                   \  }
}}|�St        |t        «      rA|j                  j	                  | j
                  «      }|	r|j                  }n|j                  }n|}|	r|n|}|	�r%t        | d«      st        d«      ‚|�[rY| j                  |«      }j                  | j
                     j                  }|j                  | j
                     j                  }�nQ| j                  |«      }| j                  |«      j                  | j                   d¬«      \  }}|j#                  |
d| j$                  | j&                  «      j)                  dd«      }|j#                  |
d| j$                  | j&                  «      j)                  dd«      }n | j                  |«      j                  | j                   d¬«      \  }}}|j#                  |
d| j$                  | j&                  «      j)                  dd«      }|j#                  |
d| j$                  | j&                  «      j)                  dd«      }|�;j+                  ||| j
                  «      \  }}|	rd|j                  | j
                  <   |j#                  |
|| j$                  | j&                  «      j)                  dd«      }| j,                  r| j/                  ||||«      \  }}n| j1                  ||||«      \  }}| j3                  || j$                  | j&                  «      }| j5                  |«      }| j7                  |«      }||fS )NrX   z«If class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `ImageGPTAttention(..., is_cross_attention=True)`.rF   re   r.   r   T)ÚshapeÚ
isinstancer   Ú
is_updatedÚgetr@   Úcross_attention_cacheÚself_attention_cacheÚhasattrrS   rX   ÚlayersÚkeysÚvaluesrW   ÚsplitrR   rM   rP   rQ   rh   ÚupdaterV   r‹   ry   r”   rY   r^   )r'   r•   r–   rr   r—   r˜   r™   rš   Úkwargsr?   rƒ   Úseq_lenr†   rž   Úcurr_past_key_valuesÚcurrent_statesro   rp   rq   rx   rs   s                        r)   r4   zImageGPTAttention.forwardÈ   sÔ  € ð 3¸$Ð>ÐØ'×-Ñ-‰ˆˆW�aàÐ!Ü˜*Ô&9Ô:Ø'×2Ñ2×6Ñ6°t·~±~ÓF�
Ù%à+5×+KÑ+KÑ(à+5×+JÑ+JÑ(à'1Ð$á2DÑ.È-ˆÚÜ˜4 Ô*Ü ðtóð ð
 Ð%©*àŸ™ MÓ2�Ø*×1Ñ1°$·.±.ÑA×FÑF�Ø,×3Ñ3°D·N±NÑC×JÑJ’àŸ™ MÓ2�Ø!Ÿ[™[¨Ó8×>Ñ>¸t¿¹ÐTUÐ>ÓV‘
��UØ—h‘h˜s B¨¯©¸¿¹ÓF×PÑPÐQRÐTUÓV�ØŸ
™
 3¨¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZ‘à $§¡¨NÓ ;× AÑ AÀ$Ç/Á/ÐWXÐ AÓ YÑˆE�3˜Ø—(‘(˜3  D§N¡N°D·M±MÓB×LÑLÈQÐPQÓRˆCØ—J‘J˜s B¨¯©¸¿¹ÓF×PÑPÐQRÐTUÓVˆEàÐ!à-×4Ñ4°S¸%ÀÇÁÓP‰JˆC�á!Ø8<�
×%Ñ% d§n¡nÑ5à—
‘
˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓWˆà×'Ò'Ø(,×(GÑ(GÈÈsÐTYÐ[iÓ(jÑ%ˆK™à(,¯
©
°5¸#¸uÀnÓ(UÑ%ˆK˜à×'Ñ'¨°T·^±^ÀTÇ]Á]ÓSˆØ—k‘k +Ó.ˆØ×(Ñ(¨Ó5ˆà˜LÐ(Ð(r*   )FNr    ©NNNNFF)r5   r6   r7   rL   r9   r"   ry   r‹   r‘   r”   r$   r%   r	   r8   r4   r;   r<   s   @r)   r>   r>   =   sÓ   ø„ ñ&<°4¸$±;ð &<ÐSVÐY]ÑS]õ &<óP )óD.)ò`*ò&ð $(Ø.2Ø59Ø6:Ø!&Ø).ñB)à—|‘|ðB)ð ˜D‘LðB)ð Ÿ™ tÑ+ð	B)ð
  %Ÿ|™|¨dÑ2ðB)ð !&§¡¨tÑ 3ðB)ð ˜$‘;ðB)ð   $™;ðB)ð 
÷B)r*   r>   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚImageGPTMLPc                 óö   •— t         ‰| �  «        |j                  }t        ||«      | _        t        ||«      | _        t        |j                     | _        t        j                  |j                  «      | _        y r    )r!   r"   r   r   Úc_fcrY   r   Úactivation_functionÚactr   rZ   r]   Údropout)r'   Úintermediate_sizerG   rN   r(   s       €r)   r"   zImageGPTMLP.__init__  s_   ø€ Ü‰ÑÔØ×&Ñ&ˆ	ÜÐ,¨iÓ8ˆŒ	Ü˜YÐ(9Ó:ˆŒÜ˜&×4Ñ4Ñ5ˆŒÜ—z‘z &×"4Ñ"4Ó5ˆ�r*   r•   r,   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S r    )r°   r²   rY   r³   )r'   r•   s     r)   r4   zImageGPTMLP.forward  s@   € ØŸ	™	 -Ó0ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr*   )r5   r6   r7   r"   r$   r%   r4   r;   r<   s   @r)   r®   r®     s#   ø„ ô6ð U§\¡\ð °e·l±l÷ r*   r®   c                   óÊ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 	 ddej
                  dedz  dej
                  dz  dej
                  dz  dej
                  dz  dedz  d	edz  d
efd„Z	ˆ xZ
S )ÚImageGPTBlockNc                 ó   •— t         ‰| �  «        |j                  }|j                  �|j                  nd|z  }t	        ||j
                  ¬«      | _        t        ||¬«      | _        t	        ||j
                  ¬«      | _	        |j                  r/t        |d|¬«      | _        t	        ||j
                  ¬«      | _        t        ||«      | _        y )Né   ©r   ©r@   T)r?   r@   )r!   r"   r   Ún_innerr   Úlayer_norm_epsilonÚln_1r>   ÚattnÚln_2Úadd_cross_attentionÚcrossattentionÚln_cross_attnr®   Úmlp)r'   rG   r@   r   Ú	inner_dimr(   s        €r)   r"   zImageGPTBlock.__init__  s¥   ø€ Ü‰ÑÔØ×(Ñ(ˆØ&,§n¡nÐ&@�F—N’NÀaÈ+Áoˆ	ä% k°v×7PÑ7PÔQˆŒ	Ü% f¸	ÔBˆŒ	Ü% k°v×7PÑ7PÔQˆŒ	à×%Ò%Ü"3°FÈtÐ_hÔ"iˆDÔÜ!2°;ÀF×D]ÑD]Ô!^ˆDÔä˜y¨&Ó1ˆ�r*   r•   r–   rr   r—   r˜   r™   rš   r,   c                 ó‚  — |}	| j                  |«      }| j                  |||||¬«      }
|
d   }|
dd  }||	z   }|�Wt        | d«      st        d| › d�«      ‚|}	| j	                  |«      }| j                  ||||||¬«      }|d   }|	|z   }||dd  z   }|}	| j                  |«      }| j                  |«      }|	|z   }|f|z   S )N)r–   rr   r™   rš   r   r   rÂ   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r–   rr   r—   r˜   rš   )r¾   r¿   r¢   rS   rÃ   rÂ   rÀ   rÄ   )r'   r•   r–   rr   r—   r˜   r™   rš   r¨   ÚresidualÚattn_outputsrx   ÚoutputsÚcross_attn_outputsÚfeed_forward_hidden_statess                  r)   r4   zImageGPTBlock.forward.  s.  € ð !ˆØŸ	™	 -Ó0ˆØ—y‘yØØ!Ø)ØØ/ð !ó 
ˆð # 1‘oˆØ˜q˜rÐ"ˆà# hÑ.ˆà Ð,ä˜4Ð!1Ô2Ü Ø=¸d¸Vð DZð Zóð ð %ˆHØ ×.Ñ.¨}Ó=ˆMØ!%×!4Ñ!4ØØ%Ø-Ø&;Ø'=Ø"3ð "5ó "Ðð -¨QÑ/ˆKà$ {Ñ2ˆMØÐ 2°1°2Ð 6Ñ6ˆGà ˆØŸ	™	 -Ó0ˆØ%)§X¡X¨mÓ%<Ð"à Ð#=Ñ=ˆàÐ 'Ñ)Ð)r*   r    r¬   )r5   r6   r7   r"   r$   r%   r	   rL   r8   r4   r;   r<   s   @r)   r·   r·     sš   ø„ õ2ð$ $(Ø.2Ø59Ø6:Ø!&Ø).ñ5*à—|‘|ð5*ð ˜D‘Lð5*ð Ÿ™ tÑ+ð	5*ð
  %Ÿ|™|¨dÑ2ð5*ð !&§¡¨tÑ 3ð5*ð ˜$‘;ð5*ð   $™;ð5*ð 
÷5*r*   r·   c                   óh   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	 e
j                  «       ˆ fd„«       Zˆ xZS )ÚImageGPTPreTrainedModelrG   ÚtransformerÚ	input_ids)ÚimageTr·   c           
      ó^  •— t         ‰| �  |«       t        |t        «      r||j	                  «       D ]h  \  }}d|v sŒd|v sŒt        j                  |d| j                  j                  t        j                  d| j                  j                  z  «      z  ¬«       Œj yt        |t        «      r�|j                  j                  }t        j                  |j                  t!        j"                  t!        j$                  ||ft         j&                  ¬«      «      j)                  dd||«      «       yy)	zInitialize the weights.rY   r&   g        rF   )r2   ÚstdrC   r   N)r!   Ú_init_weightsr�   r   Únamed_parametersÚinitÚnormal_rG   Úinitializer_rangeÚmathr1   Ún_layerr>   rH   Úcopy_rB   r$   rJ   rK   rL   rM   )r'   ÚmoduleÚnameÚpr_   r(   s        €r)   rÓ   z%ImageGPTPreTrainedModel._init_weightso  sñ   ø€ ô 	‰Ñ˜fÔ%ô �fœoÔ.Ø!×2Ñ2Ó4ò v‘��aØ˜tÒ#¨°DÒ(8ä—L‘L ¨°$·+±+×2OÑ2OÔRV×R[ÑR[Ð\]Ð`d×`kÑ`k×`sÑ`sÑ\sÓRtÑ2tÖuñvô ˜Ô 1Ô2Ø"ŸM™M×AÑAˆMÜ�J‰JØ—‘Ü—
‘
œ5Ÿ:™: }°mÐ&DÌEÏJÉJÔWÓX×]Ñ]Ø�q˜-¨óõð 3r*   )r5   r6   r7   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesr$   Úno_gradrÓ   r;   r<   s   @r)   rÍ   rÍ   f  sC   ø… àÓØ%ÐØ!€OØ!ÐØ&*Ð#Ø(Ð)Ðà€U‡]�]ƒ_óó ôr*   rÍ   c                   ón  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  de
dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dedeez  fd„«       Zˆ xZS )ÚImageGPTModelrG   c           	      óZ  •— t         ‰| �  |«       |j                  | _        t	        j
                  |j                  | j                  «      | _        t	        j
                  |j                  | j                  «      | _	        t	        j                  |j                  «      | _        t	        j                  t        |j                  «      D �cg c]  }t!        ||¬«      ‘Œ c}«      | _        t%        | j                  |j&                  ¬«      | _        d| _        | j-                  «        y c c}w )Nr»   rº   F)r!   r"   r   rN   r   Ú	EmbeddingÚ
vocab_sizeÚwterH   ÚwperZ   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚnum_hidden_layersr·   Úhr   r½   Úln_fÚgradient_checkpointingÚ	post_init)r'   rG   Úir(   s      €r)   r"   zImageGPTModel.__init__‹  sÉ   ø€ Ü‰Ñ˜Ô à×+Ñ+ˆŒä—<‘< × 1Ñ 1°4·>±>ÓBˆŒÜ—<‘< × >Ñ >ÀÇÁÓOˆŒä—J‘J˜v×0Ñ0Ó1ˆŒ	Ü—‘ÌEÐRX×RjÑRjÓLkÖlÀq¤¨fÀÖ BÒlÓmˆŒÜ% d§n¡n¸&×:SÑ:SÔTˆŒ	à&+ˆÔ#à�‰Õùò  ms   Ã
D(c                 ó   — | j                   S r    ©rê   )r'   s    r)   Úget_input_embeddingsz"ImageGPTModel.get_input_embeddings›  s   € Ø�x‰xˆr*   c                 ó   — || _         y r    r÷   )r'   Únew_embeddingss     r)   Úset_input_embeddingsz"ImageGPTModel.set_input_embeddingsž  s	   € Ø!ˆ�r*   NrÏ   Úpast_key_valuesrr   Útoken_type_idsÚposition_idsÚinputs_embedsr—   r˜   r™   rš   Úoutput_hidden_statesÚreturn_dictr¨   r,   c           
      ó  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|	�|	n| j                   j                  }	|�|n| j                   j                  }|�|�t        d«      ‚|�G| j                  ||«       |j                  «       }|j                  d|d   «      }|j                  d   }n0|�#|j                  «       dd }|j                  d   }nt        d«      ‚|�|j                  n|j                  }| j                  r%| j                  r|	rt        j                  d«       d}	|�|j                  d|d   «      }|	r|€t        | j                   ¬«      }|€B|�|j!                  «       nd}t#        j$                  |d   |¬	«      |z   }|j'                  d«      }|�|j                  |d«      }|€| j)                  |«      }| j+                  |«      }||j-                  |j                  «      z   }t/        | j                   d
d«      rt1        | j                   |||¬«      }nt3        | j                   ||¬«      }|�t3        | j                   |||¬«      }|�| j)                  |«      }||z   }| j5                  |«      }||j                  d«      fz   }|
rdnd}|
r| j                   j6                  rdnd}|rdnd}t9        | j:                  «      D ]M  \  }}|r||fz   } |||||||	|
¬«      }|d   }|
sŒ%||d   fz   }| j                   j6                  sŒE||d   fz   }ŒO | j=                  |«      } |j                  |Ž }|r||fz   }|st?        d„ |||||fD «       «      S tA        |||||¬«      S )a;  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details.

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, ImageGPTModel
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small")
        >>> model = ImageGPTModel.from_pretrained("openai/imagegpt-small")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> last_hidden_states = outputs.last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer.   r   z5You have to specify either input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rG   )rd   Ú
is_decoder)rG   rÿ   rr   rü   )rG   rÿ   rr   )rG   rÿ   rr   r—   © )r˜   r™   rš   r   rF   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr    r  )Ú.0Úvs     r)   ú	<genexpr>z(ImageGPTModel.forward.<locals>.<genexpr>=  s   è ø€ ò àØ�=ô ñùs   ‚)Úlast_hidden_staterü   r•   Ú
attentionsÚcross_attentions)!rG   rš   r   r™   r  rS   Ú%warn_if_padding_and_no_attention_maskri   rM   rœ   rd   ró   ÚtrainingÚloggerÚwarning_oncer
   Úget_seq_lengthr$   ÚarangeÚ	unsqueezerê   rë   ÚtoÚgetattrr   r   rí   rÁ   Ú	enumeraterñ   rò   r8   r   )r'   rÏ   rü   rr   rý   rþ   rÿ   r—   r˜   r™   rš   r   r  r¨   Úinput_shapeÚ
batch_sizerd   Úpast_seen_tokensÚposition_embedsr•   Útoken_type_embedsÚoutput_shapeÚall_self_attentionsÚall_cross_attentionsÚall_hidden_statesrõ   ÚblockrÉ   s                               r)   r4   zImageGPTModel.forward¡  sí  € ð` 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;ˆIØ"Ÿ™¨Ñ+‰JØÐ&Ø'×,Ñ,Ó.¨s°Ð3ˆKØ&×,Ñ,¨QÑ/‰JäÐTÓUÐUà%.Ð%:�×!Ò!À×@TÑ@Tˆà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ%Ø+×0Ñ0°°[À±_ÓEˆNá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨°B©ÀÔGÐJZÑZˆLØ'×1Ñ1°!Ó4ˆLàÐ%Ø+×0Ñ0°¸RÓ@ˆNàÐ Ø ŸH™H YÓ/ˆMØŸ(™( <Ó0ˆØ%¨×(:Ñ(:¸=×;OÑ;OÓ(PÑPˆä�4—;‘; ¨eÔ4Ü/Ø—{‘{Ø+Ø-Ø /ô	‰Nô 7Ø—{‘{Ø+Ø-ôˆNð "Ð-Ü%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"ð Ð%Ø $§¡¨Ó 8ÐØ)Ð,=Ñ=ˆMàŸ	™	 -Ó0ˆØ" m×&8Ñ&8¸Ó&<Ð%>Ñ>ˆá$5™b¸4ÐÙ%6¸4¿;¹;×;ZÒ;Z™rÐ`dÐÙ"6™B¸DÐÜ! $§&¡&Ó)ò 	P‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!áØØØØ%Ø'=Ø#Ø"3ôˆGð $ A™JˆMÚ Ø&9¸WÀQ¹Z¸MÑ&IÐ#Ø—;‘;×2Ó2Ø+?À7È1Á:À-Ñ+OÑ(ð%	Pð( Ÿ	™	 -Ó0ˆØ*˜×*Ñ*¨LÐ9ˆñ  Ø 1°]Ð4DÑ DÐáÜñ à'¨Ð:KÐM`ÐbvÐwôó ð ô 9Ø+Ø+Ø+Ø*Ø1ô
ð 	
r*   )NNNNNNNNNNNN)r5   r6   r7   r   r"   rø   rû   r   r$   r%   r	   rL   r   r8   r   r4   r;   r<   s   @r)   ræ   ræ   ‰  sF  ø„ ð˜~õ ò ò"ð ð *.Ø(,Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø!%Ø)-Ø,0Ø#'ñg
à—<‘< $Ñ&ðg
ð  ™ðg
ð Ÿ™ tÑ+ð	g
ð
 Ÿ™ tÑ+ðg
ð —l‘l TÑ)ðg
ð —|‘| dÑ*ðg
ð  %Ÿ|™|¨dÑ2ðg
ð !&§¡¨tÑ 3ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð ðg
ð 
Ð:Ñ	:òg
ó ôg
r*   ræ   z‹
    The ImageGPT Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc            !       óŠ  ‡ — e Zd ZddiZdefˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  de	dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  de
dz  de
dz  de
dz  dedeez  fd„«       Zˆ xZS )ÚImageGPTForCausalImageModelingzlm_head.weightztransformer.wte.weightrG   c                 óÌ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  dz
  d¬«      | _        | j                  «        y )Nr   F©rB   )
r!   r"   ræ   rÎ   r   ÚLinearÚn_embdré   Úlm_headrô   ©r'   rG   r(   s     €r)   r"   z'ImageGPTForCausalImageModeling.__init__U  sL   ø€ Ü‰Ñ˜Ô Ü(¨Ó0ˆÔÜ—y‘y §¡°×0AÑ0AÀAÑ0EÈEÔRˆŒð 	�‰Õr*   NrÏ   rü   rr   rý   rþ   rÿ   r—   r˜   Úlabelsr™   rš   r   r  r¨   r,   c                 ó$  — |�|n| j                   j                  }| j                  |||||||||
|||¬«      }|d   }| j                  |«      }d}|	�r|ddd…dd…f   j	                  «       }|	ddd…f   j	                  «       }t        «       } ||j                  d|j                  d«      «      |j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  |j                  ¬«      S )a&
  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, ImageGPTForCausalImageModeling
        >>> import torch
        >>> import matplotlib.pyplot as plt
        >>> import numpy as np

        >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small")
        >>> model = ImageGPTForCausalImageModeling.from_pretrained("openai/imagegpt-small")
        >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        >>> model.to(device)  # doctest: +IGNORE_RESULT

        >>> # unconditional generation of 8 images
        >>> batch_size = 4
        >>> context = torch.full((batch_size, 1), model.config.vocab_size - 1)  # initialize with SOS token
        >>> context = context.to(device)
        >>> output = model.generate(
        ...     input_ids=context, max_length=model.config.n_positions + 1, temperature=1.0, do_sample=True, top_k=40
        ... )

        >>> clusters = image_processor.clusters
        >>> height = image_processor.size["height"]
        >>> width = image_processor.size["width"]

        >>> samples = output[:, 1:].detach().cpu().numpy()
        >>> samples_img = [
        ...     np.reshape(np.rint(127.5 * (clusters[s] + 1.0)), [height, width, 3]).astype(np.uint8) for s in samples
        ... ]  # convert color cluster tokens back to pixels
        >>> f, axes = plt.subplots(1, batch_size, dpi=300)

        >>> for img, ax in zip(samples_img, axes):  # doctest: +IGNORE_RESULT
        ...     ax.axis("off")
        ...     ax.imshow(img)
        ```N)rü   rr   rý   rþ   rÿ   r—   r˜   r™   rš   r   r  r   .r.   r   )ÚlossÚlogitsrü   r•   r
  r  )rG   r  rÎ   r'  r“   r   rM   ri   r   rü   r•   r
  r  )r'   rÏ   rü   rr   rý   rþ   rÿ   r—   r˜   r)  r™   rš   r   r  r¨   Útransformer_outputsr•   Ú	lm_logitsr+  Úshift_logitsÚshift_labelsÚloss_fctÚoutputs                          r)   r4   z&ImageGPTForCausalImageModeling.forward]  sL  € ðJ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø)Ø%Ø'Ø"7Ø#9ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÑ.ˆà—L‘L Ó/ˆ	àˆØÐà$ S¨#¨2¨#ªq [Ñ1×<Ñ<Ó>ˆLØ! # q¡r '™?×5Ñ5Ó7ˆLä'Ó)ˆHÙ˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÓHÈ,×J[ÑJ[Ð\^ÓJ_Ó`ˆDáØ�\Ð$7¸¸Ð$;Ñ;ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5Ø0×AÑAô
ð 	
r*   )NNNNNNNNNNNNN)r5   r6   r7   Ú_tied_weights_keysr   r"   r   r$   r%   r	   rL   r   r8   r   r4   r;   r<   s   @r)   r"  r"  L  sa  ø„ ð +Ð,DÐEÐð˜~õ ð ð *.Ø(,Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø&*Ø!%Ø)-Ø,0Ø#'ñl
à—<‘< $Ñ&ðl
ð  ™ðl
ð Ÿ™ tÑ+ð	l
ð
 Ÿ™ tÑ+ðl
ð —l‘l TÑ)ðl
ð —|‘| dÑ*ðl
ð  %Ÿ|™|¨dÑ2ðl
ð !&§¡¨tÑ 3ðl
ð —‘˜tÑ#ðl
ð ˜$‘;ðl
ð   $™;ðl
ð # T™kðl
ð ˜D‘[ðl
ð ðl
ð  
Ð2Ñ	2ò!l
ó ôl
r*   r"  zË
    The ImageGPT Model transformer with an image classification head on top (linear layer).
    [`ImageGPTForImageClassification`] average-pools the hidden states in order to do the classification.
    c                   óB  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de	dz  de	dz  de	dz  de	dz  de
deez  fd„«       Zˆ xZS )ÚImageGPTForImageClassificationrG   c                 óè   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  | j                  d¬«      | _        | j                  «        y )NFr$  )
r!   r"   Ú
num_labelsræ   rÎ   r   r%  r&  Úscorerô   r(  s     €r)   r"   z'ImageGPTForImageClassification.__init__Ô  sR   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ(¨Ó0ˆÔÜ—Y‘Y˜vŸ}™}¨d¯o©oÀEÔJˆŒ
ð 	�‰Õr*   NrÏ   rü   rr   rý   rþ   rÿ   r)  r™   rš   r   r  r¨   r,   c                 ó„  — |�|n| j                   j                  }| j                  ||||||||	|
|¬«
      }|d   }|j                  d¬«      }| j	                  |«      }d}|�| j                  ||| j                   «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )aÊ  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, ImageGPTForImageClassification
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("openai/imagegpt-small")
        >>> model = ImageGPTForImageClassification.from_pretrained("openai/imagegpt-small")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        ```N)	rü   rr   rý   rþ   rÿ   r™   rš   r   r  r   r   re   )r+  r,  rü   r•   r
  )
rG   r  rÎ   r2   r8  Úloss_functionr   rü   r•   r
  )r'   rÏ   rü   rr   rý   rþ   rÿ   r)  r™   rš   r   r  r¨   r-  r•   Úpooled_hidden_statesr,  r+  r2  s                      r)   r4   z&ImageGPTForImageClassification.forwardÝ  sý   € ðf &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÑ.ˆà,×1Ñ1°aÐ1Ó8Ðà—‘Ð0Ó1ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDáØ�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä/ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r*   )NNNNNNNNNNN)r5   r6   r7   r   r"   r   r$   r%   r	   rL   r   r8   r   r4   r;   r<   s   @r)   r5  r5  Í  s%  ø„ ð˜~õ ð ð *.Ø(,Ø.2Ø.2Ø,0Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñT
à—<‘< $Ñ&ðT
ð  ™ðT
ð Ÿ™ tÑ+ð	T
ð
 Ÿ™ tÑ+ðT
ð —l‘l TÑ)ðT
ð —|‘| dÑ*ðT
ð —‘˜tÑ#ðT
ð ˜$‘;ðT
ð   $™;ðT
ð # T™kðT
ð ˜D‘[ðT
ð ðT
ð 
Ð1Ñ	1òT
ó ôT
r*   r5  )r"  r5  ræ   rÍ   )5Ú__doc__rØ   Útypingr   r$   r   Útorch.nnr   Ú r   rÕ   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úutils.genericr   Úconfiguration_imagegptr   Ú
get_loggerr5   r  ÚModuler   r>   r®   r·   rÍ   ræ   r"  r5  Ú__all__r  r*   r)   ú<module>rN     sB  ðñ %ã Ý ã Ý Ý %å &Ý !ß CÑ CÝ )ß JÝ 9÷ñ õ
 .Ý #÷ñ õ
 ,Ý 2ð 
ˆ×	Ñ	˜HÓ	%€ô
˜Ÿ	™	ô 
ôM)˜Ÿ	™	ô M)ô`�"—)‘)ô ô"E*Ð.ô E*ðP ô˜oó ó ððD ô
Ð+ó 
ó ð
ñD ðôôx
Ð%<¸oó x
óðx
ñv ðôô_
Ð%<ó _
óð_
òD�r*   