^j1dZddlZddlZddlZddlZddlmZmZddlm Z ddl m Z ddl m Z ddlmZddlZddlmZmZdd lmZdd lmZdd lmZmZd d lmZd dlmZd dlm Z m!Z!m"Z"d dl#m$Z$d dl%m&Z&m'Z'd dl(m)Z)d dl*m+Z+d dl,m-Z-m.Z.m/Z/ej`e1Z2e.de Gdde-Z3dKdZ4dLdZ5 dMdZ6GddejnZ8e.Gdde+Z9Gd d!ejnZ:Gd"d#ejnZ;Gd$d%ejnZ<Gd&d'ejnZ=Gd(d)ejnZ>Gd*d+ejnZ?Gd,d-ejnZ@Gd.d/e)ZAGd0d1ejnZBe>jZDd2ZEGd3d4ejneZFGd5d6eFZGGd7d8eFZHGd9d:eFZIGd;ded?e eFfd@ZLGdAdBe9ZMe.GdCdDe9ZNe.dEGdFdGe9e$ZOe.GdHdIe9ZPgdJZQy)NzPyTorch UDOP model.N)ABCabstractmethod)Sequence)deepcopy) dataclass)Any)Tensornn)CrossEntropyLoss) UdopConfig)Seq2SeqLMOutputSeq2SeqModelOutput)initialization)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer)PreTrainedModel) ModelOutputauto_docstringis_torchdynamo_compilingz Class for the model's outputs that may also contain a past key/values (to speed up sequential decoding). Includes an additional attention mask. ) custom_introceZdZUdZdZej dzed<dZej dzed<dZ e dzed<dZ e ej dzed<dZ e ej dzed<dZe ej dzed<y) BaseModelOutputWithAttentionMaska last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output. attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Attention mask used in the model's forward pass to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the weighted average in the cross-attention heads. Nlast_hidden_stateattention_maskpast_key_values hidden_states attentionscross_attentions)__name__ __module__ __qualname____doc__r torch FloatTensor__annotations__r!r"rr#tupler$r%q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/udop/modeling_udop.pyrr5s:37u((4/6/3NE%%,3$(OUT\(59M5**+d2926Je''(4/68<eE--.5q&A1 E #2  % %&>q&A1 E O OPQST U !"  $ $%=a%@! D !"  $ $%=a%@! D N NqRS T  *..r15 r/c6t|tjr|jd}n t |}tj |}||z }|dkDrAtj |g|zj|}tj||gd}|d|S)Nrr5) isinstancer*r shapelentensorr9tocat)seq target_len pad_valuenmrets r0 pad_sequencerQys#u||$ IIaL Hll3QA1ukk9+/*--c2iic * { r/c  |} tjtj|dddddf|dddddfzdz | zjd| dz } tjtj|dddddf|dddddfzdz | zjd| dz | z} | | z} |j tj }|j ddk(|j dd k(z} tj|d| jdjdd|jd}d|| <||z }tj|dddddfd j}tjtjt| dddfjd| jddddddfj | | dddddfgd }|j!dd}t#|\}}d |||f<t%t|Dcgc] }||||}}|Yt'|| }|jdj|jddd}|j |j(}t%t|Dcgc] }||||}}|H|Dcgc]=}tj*|jd|j,|j(?}}|dk(r|jd}n||jdz }tj.|Dcgc]%}t1||tj2|d'c}}tj.|Dcgc]%}t1||tj2|d'c}}|Dtj.Dcgc]%}t1||tj2|d'c}}tj||gd}tj||gd}|tj|gd}|||fScc}wcc}wcc}wcc}wcc}wcc}w)a Combine the image and text embeddings for the input to the encoder/decoder of UDOP. First, the image embeddings are created by checking for each visual patch if it is inside the bounding box of a token. If it is, the visual patch is combined with the token embedding. Then, the visual bounding boxes are combined with the text bounding boxes. Finally, the visual bounding boxes are combined with the text attention mask. Nrg@r3rr4r2Tr5F)r=r>dtypedevice)rr)r*clipfloorlongrIfloat64meangather unsqueezer:size full_likeboolrJr8rGflattenziprangerCrWonesrVr9rQ zeros_like)image_embeddings inputs_embedsbbox visual_bboxr! num_patchesmax_lenr=r>sequence_length ocr_points_x ocr_points_y ocr_points target_segrepeated_vision_embeds patch_indsindrowscolsiinput_vision_patchesitemvisual_attention_maskinputs_vision_patchess r0combine_image_text_embeddingsr|sK&"O:: T!Q']T!Q']2c9OKLQQSUVXgjkXkL  5;;Q1W Q1W =DVW\\^`acruvcvw   ,J 775== !D))B-3&499R=C+?@J"\\!Z11"5<__% S !++A.556F6K6KA6NPQSTU !nn%5%<%<= :?J:PQQ;q>*Q-0QKQ!kv! cgEJJtyy|>+?+?H]H] ^! ! !|"''*M..q11!KK[opSWdGU%5%56Ft6L%M Np++epq]a|D'5;K;KDQUJ;WXqrK! % ]r sUY\$)9)9.:N)O P s! II}.CDaHM 99dK(! ,D!N4I#JAN $ ..?`R!  qq ts%9Q$>Q)AQ.*Q3*Q8$*Q=c(eZdZdZfdZdZxZS)UdopPatchEmbeddingsz2D Image to Patch Embeddingsct||j|j}}|j|j }}t |tjjr|n||f}t |tjjr|n||f}|d|dz|d|dzz}||_||_||_||_ tj|||||_ y)Nr3r) kernel_sizestride)super__init__r=r> num_channels hidden_sizerE collectionsabcIterablerkr Conv2dproj)selfconfigr=r>rrrk __class__s r0rzUdopPatchEmbeddings.__init__s !'!2!2F4E4EJ $*$7$79K9Kk #-j+//:R:R#SZZdfpYq #-j+//:R:R#SZZdfpYq !!} 1 5*Q-:VW=:XY $$(&IIlKZXbc r/c 8|j\}}}}||jdk7s||jdk7r2td|d|d|jdd|jdd |j|}|j dj dd}|S)Nrr3zInput image size (*z) doesn't match model (z).rS)rFr= ValueErrorrrbr;)r pixel_values batch_sizerheightwidth embeddingss r0forwardzUdopPatchEmbeddings.forwards2>2D2D/ L&% T__Q' '5DOOA4F+F$VHAeW4KDOO\]L^K__`aeapapqras`ttvw YY|, ''*44Q: r/)r&r'r(r)rr __classcell__rs@r0r~r~s& dr/r~cneZdZUeed<dZdZdZdZdgZ e jfdZ dZ xZS) UdopPreTrainedModelr transformer)imagetextTFwoc t|||jj}t |t r$t j|j|dzyt |tjrOt j|jd||j t j|jyyt |tr_|jj}|jj}t j |j"jd||dzzyt |t$r0t j |j&jd|dzyt |t(rTt+|drG|jj,s0t j |j.jd|dzyyyt |t0r9t j |j2jd||jjdzzt+|j2dr?|j2j)t j|j2jt j |j4jd||jj6dzzt+|j4drA|j4j*t j|j4jyyyt |t8rt j |j:jd||jjdzzt+|j:dr?|j:j)t j|j:jt j |j<jd||jjdzzt+|j<dr?|j<j)t j|j<jt j |j4jd||jj6dzzt+|j4drA|j4j*t j|j4jyyyt |t>rP|jj}|jj@}|jjB}t j |jDjd|||zdzzt j |jFjd||dzzt j |jHjd||dzzt j |jJjd|||zdzz|jLr3t j |j"jd||dzzyyy)zInitialize the weightsr2rT)r\stdNlm_headbias)'r _init_weightsrinitializer_factorrE UdopLayerNorminit constant_weightr r trunc_normal_rzeros_RelativePositionBiasBased_modelnormal_relative_attention_bias UdopModelsharedUdopForConditionalGenerationhasattrtie_word_embeddingsrUdopDenseActDensewird_ffUdopDenseGatedActDensewi_0wi_1 UdopAttentiond_kv num_headsqkvohas_relative_attention_bias)rmodulefactorrkey_value_proj_dimn_headsrs r0rz!UdopPreTrainedModel._init_weightssS f%// fm , NN6==&3, 7  *   v}}3F C{{& FKK('  8 9[[33Fkk))G LL77>>SfY`eiXiNj k  * LL--CVc\ J  < =vy)$++2Q2Q V^^22&3,O3R)  1 2 LL))DKKDWDW\`C`9a bvyy&)fiinn.H FIINN+ LL))DKKDTDTY]C]9^ _vyy&)fiinn.H FIINN+/I)  6 7 LL++#6dkkFYFY^bEb;c dv{{F+ 0@0@0L FKK,,- LL++#6dkkFYFY^bEb;c dv{{F+ 0@0@0L FKK,,- LL))DKKDTDTY]C]9^ _vyy&)fiinn.H FIINN+/I)  .kk))G!%!1!1 kk++G LLs7M_C_dhBh8i j LLs'4-8P Q LLs'4-8P Q LLs7M_C_dhBh8i j11 V;;BBRX]dim\mRno2/r/c|jj}|jj}|Jd|j|j}|dddfj |dddf<||d<|Jd|j |dk(|tj|dk\jsJd |S) Nzself.model.config.decoder_start_token_id has to be defined. In Udop it is usually set to the pad_token_id. See Udop docs for more information.r4r3).rz1self.model.config.pad_token_id has to be defined.rz8Verify that `shifted_input_ids` has only positive values) rdecoder_start_token_id pad_token_id new_zerosrFclone masked_fill_r*allry)r input_idsrrshifted_input_idss r0 _shift_rightz UdopPreTrainedModel._shift_right1s!%!C!C{{// %1  @ 1 &// @%.sCRCx%8%>%>%@#qr'"$:&!'\)\\'&&'8D'@,Oyy*a/0557s9ss7  r/)r&r'r(r r,base_model_prefixinput_modalitiessupports_gradient_checkpointing_can_compile_fullgraph_keep_in_fp32_modulesr*no_gradrrrrs@r0rrsK %(&*#"!FU]]_-p-p`!r/rc&eZdZdfd ZdZxZS)rct|tjt j ||_||_y)ze Construct a layernorm module in the Udop style. No bias and no subtraction of mean. N)rrr Parameterr*rervariance_epsilon)rrepsrs r0rzUdopLayerNorm.__init__Js1 ll5::k#:; #r/c|jtjjdj dd}|tj ||j zz}|jjtjtjfvr%|j|jj}|j|zS)NrSr4T)keepdim) rIr*float32powr\rsqrtrrrVfloat16bfloat16)rr#variances r0rzUdopLayerNorm.forwardRs !##EMM266q9>>r4>P% Ht?T?T4T(UU  ;;   ? ?),,T[[->->?M{{]**r/)gư>r&r'r(rrrrs@r0rrIs $ +r/rc*eZdZdeffd ZdZxZS)rrc^t|tj|j|j d|_tj|j |jd|_tj|j|_ t|j|_ yNFr)rrr LinearrrrrDropout dropout_ratedropoutr dense_act_fnactrrrs r0rzUdopDenseActDense.__init__dsn ))FNNFKKeD))FKKeDzz&"5"56 &--.r/c|j|}|j|}|j|}t|jj t jr|j|jj jk7r`|jj jt jk7r/|j|jj j}|j |}|SN) rrrrErrr*r rVint8rI)rr#s r0rzUdopDenseActDense.forwardks . /  ]3 tww~~u|| 4##tww~~';';;$$ 2),,TWW^^-A-ABM . r/r&r'r(r rrrrs@r0rrcs/z/ r/rc*eZdZdeffd ZdZxZS)rrct|tj|j|j d|_tj|j|j d|_tj|j |jd|_tj|j|_ t|j|_yr)rrr rrrrrrrrrrrrrs r0rzUdopDenseGatedActDense.__init__{s IIfnnfkkF IIfnnfkkF ))FKKeDzz&"5"56 &--.r/c,|j|j|}|j|}||z}|j|}t |j j tjr|j|j j jk7r`|j j jtjk7r/|j|j j j}|j |}|Sr) rrrrrErrr*r rVrrI)rr# hidden_gelu hidden_linears r0rzUdopDenseGatedActDense.forwardshhtyy78  -0 #m3  ]3 tww~~u|| 4##tww~~';';;$$ 2),,TWW^^-A-ABM . r/rrs@r0rrzs/z/r/rc*eZdZdeffd ZdZxZS) UdopLayerFFrct||jrt||_nt ||_t |j|j|_ tj|j|_ y)Nr)rr is_gated_actrDenseReluDenserrrlayer_norm_epsilon layer_normr rrrrs r0rzUdopLayerFF.__init__s_    "8"@D "3F";D 'F>+B+B*CD,,  4<<eD4<<eD4<<eD4>>4<=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should allow for more graceful generalization to longer sequences than the model has been trained on Args: relative_position: an int32 Tensor bidirectional: a boolean - whether the attention is bidirectional num_buckets: an integer max_distance: an integer Returns: a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) rrSr3) rIr*rZabsminrflogfloatmathr`where)relative_position bidirectional num_buckets max_distancerelative_buckets max_exactis_smallrelative_position_if_larges r0_relative_position_bucketz'UdopAttention._relative_position_buckets(,  A K !2Q!6 : :5:: F T T  % *; < !&+N>NO`>a!b b  1$ $y0&/ II'--/); <hh|i/0 1Y& ( "UZZ. &" &+YY &8RT_bcTc(d& " EKK2CE_``r/c| |jjj}tj|tj |dddf|z}tj|tj |dddf}||z }|j ||j |j|j}|j|} | jgdjd} | S)z%Compute binned relative position biasNrUrrr)rSrr3r) rrrWr*r8rZrrrr permuter^) r query_length key_lengthrWpast_seen_tokenscontext_positionmemory_positionrrelative_position_bucketvaluess r0 compute_biaszUdopAttention.compute_biass >1188??F << EJJvVWXZ^W^_brr,,zFSTXZ[T[\+.>>#'#A#A #.;;== $B$  --.FG *44Q7 r/c |jdd}g|d|j} ||j|jnd} t | t j r| jn| } |du} |j|j| jdd} d} t |trA|jj|j} | r |j}n|j}n|}| r|n|}| rK|I| rG|j |jj"}|j |jj$}ng|jddd|j}|j'|j|jdd}|j)|j|jdd}|K|j+|||j\}}| r)t |trd|j|j<t j,| |jdd}||jd }|j.s`t j0d| jd|d|f|j2|j4 }|j6r6|j8r*d|_n"|j=|d||j2| }|#|ddddddd|jd f}||z}|}||z }t>j@jC|jEd jG|}t>j@jI||jH|j8 }t j,||}|jddjK}|jLg|d}|jO|}||f}|r||fz}|S)z Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). Nr4rr3rSFTrrWrV)rWr%r5)ptraining)(rFrget_seq_lengthrrEr*r rrr<r;r is_updatedgetcross_attention_cacheself_attention_cachelayerskeysr)rrupdatematmulrzerosrWrVrr/ requires_gradr*r functionalsoftmaxrtype_asr contiguousreshaper)rr#maskkey_value_states position_biasr"output_attentionskwargs input_shape hidden_shaper%is_cross_attention query_statesr1curr_past_key_valuescurrent_states key_states value_stateskv_shapescoresr$ causal_maskposition_bias_masked attn_weights attn_outputoutputss r0rzUdopAttention.forwards$))#2. BBbB$*A*AB M\Mh?99$..Ino7ABRTYT`T`7a+113gw.T9vvm,11,?II!QO  o': ;(3377GJ!'6'L'L$'6'K'K$#2 -?)] /"=*-44T^^DIIJ/66t~~FMMLP--cr2PBP8O8OPH/44X>HHANJ66.166x@JJ1aPL*+?+F+FzS_aeaoao+p( L%*_FY*ZAEO..t~~>lJ,@,@A,FG  #))"-J33 %  **1-{1~zJSYS`S`hnhtht! ..4==26M/ $ 1 1NJv}}Wg!2! "1a,Bj.>.>r.B,B#BC - ; ,&&}},,V\\^,DLLVT }},,\T\\TXTaTa,b ll<> !++Aq1<<> )k));;;; ff[)  . /Gr/FN)T )Nr)NNNNF) r&r'r(r intr staticmethodrr*rrrs@r0rrsX%* $ , ,: ,D- - ^([r/rc>eZdZddedzffd Z ddZxZS)UdopLayerSelfAttentionNrct|t||||_t |j |j |_tj|j|_ y)Nrrr) rrr SelfAttentionrrrrr rrrrs r0rzUdopLayerSelfAttention.__init__osT * 0KW` (F *D'%'\\2Lk%Z"%'\\2Lk%Z"r/cbtj|dd}||jdz zj}|j |dddddf}|j |dddddf}|j |dddddf}|j |dddddf}||z|z|z}|S)NrTr2r3rrSr)r*rXrrZrr)rrileft_position_embeddingsupper_position_embeddingsright_position_embeddingslower_position_embeddingsrs r0rzUdopCellEmbeddings.forwardszz$S)77!;<BBD#'#=#=d1a7m#L $($>$>tAq!G}$M!$($>$>tAq!G}$M!$($>$>tAq!G}$M! %' (' (( ( r/)irrs@r0rr s[r/r)g?g?ceZdZdZ d fd Ze d dedzdeee fdzdefdZ d dedzdeee fdzdefdZ d Z d dedzdeee fdzdefd Z xZS) ra Base class of relative biases. Args: num_heads (`int`): Number of attention heads in the model, it will create embeddings of size `num_heads`, which will be added to the scores of each token pair. relative_attention_num_buckets (`int`, *optional*, defaults to 32): Pair token metric (distance in the sequence, distance in pixels etc.) will be bucketed, parameter is defining number of such buckets. bidirectional (`bool`, *optional*, defaults to `True`): Whether the distance should be bidirectional for a pair of tokens. If `False`, then distance(tok1, tok2) == distance(tok2, tok1). scaling_factor (`int`, *optional*, defaults to 1): Defining factor which will be used to scale relative distance. max_distance (`int`, *optional*, defaults to 128): All distances above this value will end up in the one/same bucket. augmentation (`bool`, *optional*, defaults to `False`): Whether to multiply relative distances by a random scalar. expand (`bool`, *optional*, defaults to `False`): Whether to expand an existing pretrained model with subsequent additions of prefix_bucket. Nc (t |||_||_||_||_||_||_||_| |_ ||_ |r|jsdnd} tj|j| z|j|_ y)NrSr)rr prefix_bucket augmentationlevelrscaling_factorrrexpandrr r r) rrrrrrrrrr extra_headrs r0rz!RelativePositionBiasBase.__init__Bs *( (,*" .L+' Q ')||D4W4WZd4dfjftft'u$r/r!rireturncyrr.)rr!ris r0 prepare_inputz&RelativePositionBiasBase.prepare_input[s r/c|j||}t||j|j|j}|S)Nr!)rget_relative_position_bucketrrr)rr!rir rp_buckets r0 get_bucketz#RelativePositionBiasBase.get_bucketcsE ..~tD8 ,,;;**  r/c|dddddf}|dddddf}||z }|jr%|jr|tjtz}||j z}|j tjSr) rr/randomuniformAUGMENTATION_RANGErrIr*rZ)r positionsr&r'rs r0get_relative_positionz.RelativePositionBiasBase.get_relative_positionmsy$Q4Z0#AtQJ/+.>>    1C!D D T000 ##EJJ//r/c|jr|jrtj|jdz|j }|j jj|jjd|jd|jj|jd||_d|_|j||}|jr|jddk(r6|jddkDr"|j|jddd}|dddddfdk}|jd}t|jjD]4\}}|j||d||df<|jdz|||dd|f<6|j |} | j!dk7r t#d| j%gd } | S) NrSg?Frr3r4r7z Wrong dimension of values tensor)rrr3rS)rrr r rrrrdatarr_r:sum enumeratecpunumpyr6rr") rr!rinew_biasr is_prefix num_prefixidxnum_prefix_rowr)s r0rz RelativePositionBiasBase.forwardws ;;4--||D$G$G!$KT^^\HJNJfJfJmJmJrJrHOO !F4#F#F GJMHOO !D!D!F G+3D (DKOOND9   ~~a A%.*=*=a*@1*D%,,^-@-@-CQJ Q1W )I"r*J'01A1G1G1I'J k#^CGCfCf #?@CGCfCfijCj #~?@ k55i@ ::<1 ?@ @ - r/) NrUTr3rVtokensFFFNN)r&r'r(r)rrr dictstrrrrrrrrs@r0rr,s.')v2)-&*   38nt#     $T#s(^VZEZfl0ftm$sCx.SWBWcir/rcLeZdZdfd ZddedzdeeefdzdefdZxZ S) RelativePositionBias1Dc *t|d||d|y)z Reimplementation of T5 relative position bias. Distance between given tokens is their distance in the sequence. Parameters are the same as in base class rrNr.rrrrrrDrs r0rzRelativePositionBias1D.__init__ \\\U[\r/Nr!rirc|jdk7r td|jtj|j dtj |jdddf}|S)Nr3zNo need to scale 1d featuresrU)rrrr*r8r_rZrW)rr!rirs r0rz$RelativePositionBias1D.prepare_inputsf   ! #;< < 66 LL,,Q/uzz.J_J_ `aeghah i ! r/)r3rVr r&r'r(rr rrrrrrs@r0rrs7]!FTM!SRUXY]H]!io!r/rcLeZdZdfd ZddedzdeeefdzdefdZxZ S) RelativePositionBiasHorizontalc *t|d||d|y)z Represents in the bucket embeddings horizontal distance between two tokens. Parameters are the same as in base class rNr.rrs r0rz'RelativePositionBiasHorizontal.__init__rr/Nr!rirc|jdkDs td| td|ddddddgfjd}|j|S)Nr2ENeed to scale the values of bboxes, as there are in small (0,1) rangez6Bbox is required for horizontal relative position biasrrSr4r5rrr\r)rr!rihorizontal_positions r0rz,RelativePositionBiasHorizontal.prepare_inputsb""S(de e <UV V&*1a!Q<&8&=&="&=&E))*=>>r/drrrrs@r0rrs7]?FTM?SRUXY]H]?io?r/rcLeZdZdfd ZddedzdeeefdzdefdZxZ S) RelativePositionBiasVerticalc *t|d||d|y)z Represents in the bucket embeddings vertical distance between two tokens. Parameters are the same as in base class rNr.rrs r0rz%RelativePositionBiasVertical.__init__rr/Nr!rirc|jdkDs td| td|ddddddgfjd}|j|S)Nr2rz4Bbox is required for vertical relative position biasr3rr4r5r)rr!rivertical_positions r0rz*RelativePositionBiasVertical.prepare_inputsb""S(de e <ST T$(A1v$6$;$;$;$C))*;<$%[\\+: K(   \)4C{C D E r/czeZdZdZfdZededefdZdZ dZ d de e zfdZ xZS) UdopStackz This class is based on `T5Stack`, but modified to take into account the image modality as well as 2D position embeddings. c t||tj|j|j |_t||_|j|_ |j|_ tjt|jDcgc]}t|t|dk(|c}|_t!|j |j"|_tj&|j(|_|js%t-|j.|j0|_|j5||_|j9ycc}w)Nrr\r)rrr r  vocab_sizer embed_tokensr~ embed_patchesr num_layersrlrdrjrablockrrfinal_layer_normrrrrrrcell_2d_embedding_get_relative_bias relative_bias post_init)rrrwrs r0rzUdopStack.__init__s   LL):):FNNK08 ++ ++]]_deietet_u vZ[Yv4Q|||j}|jj rtA|j||| }nX|ddddddf}|j#|j}d|z tjB|jjDz}|j r|tG|j|||}nd}|rdnd}| rdnd}| r|j rdnd}|j rd} n|jI||} | |z} d}|}|jK|}tM|jND]a\}} |r||fz}| ||| |||| | |  }!|!d}|!d} |j r | |!| rdnd}| sC||!dfz}|j sY||!dfz}c|jQ|}|jK|}|r||fz}|stSd ||| |||fDStU||| |||!S)"Ndecoder_zYou cannot specify both z inputs and zinputs_embeds at the same timerr4)r7rr-)r7rr7z Empty batchzYou have to specify either z inputs or rhzz$UdopStack.forward..s = s)r r!r"r#r$r%)+rr_rCoutput_hidden_statesrxrrr*numelr_r<fullrrWrVr9rfrIr warningrrr=r>r|ris_encoder_decoderrrr0rrerrsrrrrrrrr-r)"rrr!rirurvrhrrjrgrBr"r_rCrrxrDerr_msg_prefixrErkr seq_lengthpast_key_values_lengthmask_seq_lengthrOencoder_extended_attention_maskall_hidden_statesall_attentionsall_cross_attentionsrwr#rw layer_module layer_outputss" r0rzUdopStack.forward$s&"+!6IDKK+/??ZN*>*:+nEUUst  "u{{9'='A#..*K!r;r?;I  "y'<YAW[\A\ 9dkk.F.FyO_O_gpgvgvwI"[[9;K;KS\SbSbcN;;|I4D4DIOO\D#..*K!,,^AtT1:J*UcTddqrs s    ( !_`` --i8M  ##11,?   '++00DKK4J4JJK2O  && && 3 /M4(,,.s3K4#3 T33D9 9M!, J  ?? k&OPTvUj$k k? ??_4;;11&9$DKK8,dkk:Z'O'3$++&FO#OETE`!?!?!Afg  !*B*D4zAO"ZZ OML`L`aN ;; ! !,{{+- / K)D$)9:K%..}/B/B.CK, M--m<  ]3  1]4D D  ""#%"(    0+)++%1   r/NNNNNNNNNNNNNNN)r&r'r(r)rrXr rrrrr-rrrrs@r0rrs ,B:B2PBB!+  "#!!D $ 1 1%D r/rc#eZdZdddddZfdZdZdZe dd edzd edzd e e e fdzd edzd e e e fdzdedzdedzdedzdedzde dzdedzde dzde dzde dzde dzdeezf dZxZS)r shared.weightpatch_embed.proj.weightpatch_embed.proj.bias)encoder.embed_tokens.weightdecoder.embed_tokens.weight!encoder.embed_patches.proj.weightencoder.embed_patches.proj.biasczt||tj|j|j |_t||_t|}d|_ d|_ t||_ t|}d|_ |j|_t||_|j#y)NFT)rrr r rrrr~ patch_embedrrr_rencodernum_decoder_layersrdecoderrrrencoder_configdecoder_configrs r0rzUdopModel.__init__s  ll6#4#4fnnE .v6!&)$)!#(  0 !&)$(!$*$=$=! 0  r/c|jSrrrs r0get_input_embeddingszUdopModel.get_input_embeddings {{r/c~||_|jj||jj|yrrrrrrs r0rzUdopModel.set_input_embeddings -$  )).9 )).9r/Nrr!rirrjdecoder_input_idsdecoder_attention_maskrhencoder_outputsr"decoder_inputs_embedsr_rCrrxrc H| | n|jj} ||n|jj}| |j||||||| || } | d}|r | jn| d}|j ||| | ||| | || }|s;t dt|D}t dt| D} || zSt|j|j|j|j|j| j| j| jS)aA bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] token. See `pixel_values` for `patch_sequence_length`. visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. Example: ```python >>> from transformers import AutoProcessor, AutoModel >>> from datasets import load_dataset >>> import torch >>> # load model and processor >>> # in this case, we already have performed OCR ourselves >>> # so we initialize the processor with `apply_ocr=False` >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False) >>> model = AutoModel.from_pretrained("microsoft/udop-large") >>> # load an example image, along with the words and coordinates >>> # which were extracted using an OCR engine >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> inputs = processor(image, words, boxes=boxes, return_tensors="pt") >>> decoder_input_ids = torch.tensor([[model.config.decoder_start_token_id]]) >>> # forward pass >>> outputs = model(**inputs, decoder_input_ids=decoder_input_ids) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 1, 1024] ```) rr!rirrjrhrCrrxrr3 rr!rhr"rurvr_rCrrxc32K|]\}}|dk7s |ywr3Nr.rrvalues r0rz$UdopModel.forward..|#cjc5Z]abZbE#c c32K|]\}}|dk7s |ywr&r.r's r0rz$UdopModel.forward..}r)r*)r r"decoder_hidden_statesdecoder_attentionsr%encoder_last_hidden_stateruencoder_attentions)rr_rxrr!rr-rrr r"r#r$r%)rrr!rirrjrr rhr!r"r"r_rCrrxrDr#rvdecoder_outputss r0rzUdopModel.forwardsLP"+!6IDKK>> from transformers import AutoProcessor, UdopForConditionalGeneration >>> from datasets import load_dataset >>> # load model and processor >>> # in this case, we already have performed OCR ourselves >>> # so we initialize the processor with `apply_ocr=False` >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False) >>> model = UdopForConditionalGeneration.from_pretrained("microsoft/udop-large") >>> # load an example image, along with the words and coordinates >>> # which were extracted using an OCR engine >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> # one can use the various task prefixes (prompts) used during pre-training >>> # e.g. the task prefix for DocVQA is "Question answering. " >>> question = "Question answering. What is the date on the form?" >>> encoding = processor(image, question, text_pair=words, boxes=boxes, return_tensors="pt") >>> # autoregressive generation >>> predicted_ids = model.generate(**encoding) >>> print(processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]) 9/30/92 ```N rrirjrr!rhrCrrxrr3r$rr) ignore_indexr4rS) losslogitsr"r,r-r%r.rur/)rr_rxrrr!rrrrr r<r_r r"r#r$r%r )rrr!rirrjrr rhr!r"r"r_rCrrxr8rDr#rvr0sequence_output lm_logitsr<loss_fctrs r0rz$UdopForConditionalGeneration.forwards\"+!6IDKK>> from transformers import AutoProcessor, UdopEncoderModel >>> from huggingface_hub import hf_hub_download >>> from datasets import load_dataset >>> # load model and processor >>> # in this case, we already have performed OCR ourselves >>> # so we initialize the processor with `apply_ocr=False` >>> processor = AutoProcessor.from_pretrained("microsoft/udop-large", apply_ocr=False) >>> model = UdopEncoderModel.from_pretrained("microsoft/udop-large") >>> # load an example image, along with the words and coordinates >>> # which were extracted using an OCR engine >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train") >>> example = dataset[0] >>> image = example["image"] >>> words = example["tokens"] >>> boxes = example["bboxes"] >>> encoding = processor(image, words, boxes=boxes, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state ```r:)rrCrrxr) rrrir!rrjrhrCrrxrDr!s r0rzUdopEncoderModel.forwardnsv2C1N-TXT_T_TqTq$8$D $++JjJj &1%r\s #$! %# '!CC)J9-   8 $  #={#= #=L. $ P/f")):N!/N!N!d+BII+4 .RYY<"))&BIIFRYYDbii@Y *Y x8 -FF!eryy#eP!5!$?%=?&=#;=&RYY* !0, 9Q0R,k #k \] #] ] @w #6w w tl*ll^ cr/