Ë
    ÛÓ^jÛå  ã                   óB  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ ddlm Z m!Z! ddl"m#Z#  e!jH                  e%«      Z&g d¢Z' e d¬«      e G d„ de«      «       «       Z( G d„ dejR                  «      Z* G d„ dejR                  «      Z+ G d„ dejR                  «      Z, G d„ dejR                  «      Z- G d„ dejR                  «      Z. G d „ d!ejR                  «      Z/ G d"„ d#ejR                  «      Z0 G d$„ d%ejR                  «      Z1 G d&„ d'e«      Z2 G d(„ d)ejR                  «      Z3 G d*„ d+ejR                  «      Z4 G d,„ d-ejR                  «      Z5 G d.„ d/ejR                  «      Z6 G d0„ d1ejR                  «      Z7e  G d2„ d3e«      «       Z8e  G d4„ d5e8«      «       Z9 e d6¬«       G d7„ d8e8«      «       Z:e  G d9„ d:e8«      «       Z;e  G d;„ d<e8«      «       Z<e  G d=„ d>e8«      «       Z=g d?¢Z>y)@zPyTorch CANINE model.é    N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚloggingé   )ÚCanineConfig)é   é+   é;   é=   éI   éa   ég   éq   é‰   é•   é�   é­   éµ   éÁ   éÓ   éß   a  
    Output type of [`CanineModel`]. Based on [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly
    different `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
    Transformer encoders.
    )Úcustom_introc                   ó¾   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)ÚCanineModelOutputWithPoolinga§  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model (i.e. the output of the final
        shallow Transformer encoder).
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Hidden-state of the first token of the sequence (classification token) at the last layer of the deep
        Transformer encoder, further processed by a Linear layer and a Tanh activation function. The Linear layer
        weights are trained from the next sentence prediction (classification) objective during pretraining.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of each
        encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length //
        config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of each layer plus the
        initial input to each Transformer encoder. The hidden states of the shallow encoders have length
        `sequence_length`, but the hidden states of the deep encoder have length `sequence_length` //
        `config.downsampling_rate`.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape `(batch_size,
        num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length //
        config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions weights after the
        attention softmax, used to compute the weighted average in the self-attention heads.
    NÚlast_hidden_stateÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   ÚtorchÚFloatTensorÚ__annotations__r-   r.   Útupler/   © ó    úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/canine/modeling_canine.pyr+   r+   1   sh   … ñð, 37Ð�u×(Ñ(¨4Ñ/Ó6Ø.2€M�5×$Ñ$ tÑ+Ó2Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r9   r+   c                   óè   ‡ — e Zd ZdZˆ fd„Zdedefd„Zdededefd„Z	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  f
d„Zˆ xZS )ÚCanineEmbeddingsz<Construct the character, position and token_type embeddings.c           	      óÚ  •— t         ‰| �  «        || _        |j                  |j                  z  }t        |j                  «      D ]2  }d|› �}t        | |t        j                  |j                  |«      «       Œ4 t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                   «      | _        | j%                  dt'        j(                  |j*                  «      j-                  d«      d¬«       y )NÚHashBucketCodepointEmbedder_©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistent)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚnum_hash_functionsÚrangeÚsetattrr   Ú	EmbeddingÚnum_hash_bucketsÚchar_position_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr4   ÚarangeÚmax_position_embeddingsÚexpand)ÚselfrG   Úshard_embedding_sizeÚiÚnameÚ	__class__s        €r:   rF   zCanineEmbeddings.__init__Y   s   ø€ Ü‰ÑÔàˆŒð  &×1Ñ1°V×5NÑ5NÑNÐÜ�v×0Ñ0Ó1ò 	]ˆAØ1°!°Ð5ˆDÜ�D˜$¤§¡¨V×-DÑ-DÐFZÓ [Õ\ð	]ô )+¯©°V×5LÑ5LÈf×N`ÑN`Ó(aˆÔ%Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	õ 	
r9   Ú
num_hashesÚnum_bucketsc                 ó¼   — |t        t        «      kD  rt        dt        t        «      › �«      ‚t        d| }g }|D ]  }|dz   |z  |z  }|j                  |«       Œ  |S )a—  
        Converts ids to hash bucket ids via multiple hashing.

        Args:
            input_ids: The codepoints or other IDs to be hashed.
            num_hashes: The number of hash functions to use.
            num_buckets: The number of hash buckets (i.e. embeddings in each table).

        Returns:
            A list of tensors, each of which is the hash bucket IDs from one hash function.
        z`num_hashes` must be <= Nr   )ÚlenÚ_PRIMESÚ
ValueErrorÚappend)rZ   Ú	input_idsr_   r`   ÚprimesÚresult_tensorsÚprimeÚhasheds           r:   Ú_hash_bucket_tensorsz%CanineEmbeddings._hash_bucket_tensorsn   sp   € ð œœG›Ò$ÜÐ7¼¼G»°~ÐFÓGÐGä˜˜*Ð%ˆàˆØò 	*ˆEØ  1‘}¨Ñ-°Ñ<ˆFØ×!Ñ! &Õ)ð	*ð Ðr9   Úembedding_sizec                 ó  — ||z  dk7  rt        d|› d|› d�«      ‚| j                  |||¬«      }g }t        |«      D ]-  \  }}d|› �}	 t        | |	«      |«      }
|j	                  |
«       Œ/ t        j                  |d¬«      S )	zDConverts IDs (e.g. codepoints) into embeddings via multiple hashing.r   zExpected `embedding_size` (z) % `num_hashes` (z) == 0)r_   r`   r>   rC   ©Údim)rd   rk   Ú	enumerateÚgetattrre   r4   Úcat)rZ   rf   rl   r_   r`   Úhash_bucket_tensorsÚembedding_shardsr\   Úhash_bucket_idsr]   Úshard_embeddingss              r:   Ú_embed_hash_bucketsz$CanineEmbeddings._embed_hash_buckets…   s«   € à˜JÑ&¨!Ò+ÜÐ:¸>Ð:JÐJ\Ð]gÐ\hÐhnÐoÓpÐpà"×7Ñ7¸	ÈjÐfqÐ7ÓrÐØÐÜ"+Ð,?Ó"@ò 	6ÑˆAˆØ1°!°Ð5ˆDØ2œw t¨TÓ2°?ÓCÐØ×#Ñ#Ð$4Õ5ð	6ô
 �y‰yÐ)¨rÔ2Ð2r9   Nrf   Útoken_type_idsrA   Úinputs_embedsÚreturnc                 óB  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€:t        j                  |t        j                  | j                  j
                  ¬«      }|€P| j                  || j                  j                  | j                  j                  | j                  j                  «      }| j                  |«      }||z   }| j                  |«      }	||	z  }| j                  |«      }| j                  |«      }|S )NrC   r   ©ÚdtypeÚdevice)ÚsizerA   r4   ÚzerosÚlongr~   rw   rG   rH   rI   rM   rP   rN   rQ   rU   )
rZ   rf   rx   rA   ry   Úinput_shapeÚ
seq_lengthrP   Ú
embeddingsÚposition_embeddingss
             r:   ÚforwardzCanineEmbeddings.forward“   s  € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLàÐ!Ü"Ÿ[™[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNàÐ Ø ×4Ñ4Ø˜4Ÿ;™;×2Ñ2°D·K±K×4RÑ4RÐTX×T_ÑT_×TpÑTpóˆMð !%× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×;Ñ;¸LÓIÐØÐ)Ñ)ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr9   )NNNN)r0   r1   r2   r3   rF   Úintrk   rw   r4   Ú
LongTensorr5   r†   Ú__classcell__©r^   s   @r:   r<   r<   V   s®   ø„ ÙFô
ð*¸#ð ÈCó ð.3¸Sð 3Ècð 3Ð`có 3ð  .2Ø26Ø04Ø26ñ!à×#Ñ# dÑ*ð!ð ×(Ñ(¨4Ñ/ð!ð ×&Ñ&¨Ñ-ð	!ð
 ×(Ñ(¨4Ñ/ð!ð 
×	Ñ	÷!r9   r<   c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚCharactersToMoleculeszeConvert character sequence to initial molecule sequence (i.e. downsample) using strided convolutions.c                 ó>  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  ¬«      | _        t        |j                     | _	        t        j                  |j                  |j                  ¬«      | _
        y )N©Úin_channelsÚout_channelsÚkernel_sizeÚstrider?   )rE   rF   r   ÚConv1drH   Údownsampling_rateÚconvr
   Ú
hidden_actÚ
activationrQ   rR   ©rZ   rG   r^   s     €r:   rF   zCharactersToMolecules.__init__º   st   ø€ Ü‰ÑÔä—I‘IØ×*Ñ*Ø×+Ñ+Ø×0Ñ0Ø×+Ñ+ô	
ˆŒ	ô ! ×!2Ñ!2Ñ3ˆŒäŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r9   Úchar_encodingrz   c                 ó2  — |d d …dd…d d …f   }t        j                  |dd«      }| j                  |«      }t        j                  |dd«      }| j                  |«      }|d d …dd…d d …f   }t        j                  ||gd¬«      }| j                  |«      }|S )Nr   r   é   rC   rn   )r4   Ú	transposer•   r—   rr   rQ   )rZ   r™   Úcls_encodingÚdownsampledÚdownsampled_truncatedÚresults         r:   r†   zCharactersToMolecules.forwardÇ   sš   € à$¢Q¨¨!¨ªQ YÑ/ˆô Ÿ™¨°q¸!Ó<ˆØ—i‘i Ó.ˆÜ—o‘o k°1°aÓ8ˆØ—o‘o kÓ2ˆð !,ªA¨q°¨t²Q¨JÑ 7Ðô —‘˜LÐ*?Ð@ÀaÔHˆà—‘ Ó'ˆàˆr9   ©	r0   r1   r2   r3   rF   r4   ÚTensorr†   r‰   rŠ   s   @r:   rŒ   rŒ   ·   s'   ø„ ÙoôUð U§\¡\ð °e·l±l÷ r9   rŒ   c                   ó|   ‡ — e Zd ZdZˆ fd„Z	 ddej                  dej                  dz  dej                  fd„Zˆ xZS )	ÚConvProjectionz�
    Project representations from hidden_size*2 back to hidden_size across a window of w = config.upsampling_kernel_size
    characters.
    c                 ó†  •— t         ‰| �  «        || _        t        j                  |j
                  dz  |j
                  |j                  d¬«      | _        t        |j                     | _
        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )Nr›   r   rŽ   r?   )rE   rF   rG   r   r“   rH   Úupsampling_kernel_sizer•   r
   r–   r—   rQ   rR   rS   rT   rU   r˜   s     €r:   rF   zConvProjection.__init__é   s�   ø€ Ü‰ÑÔØˆŒÜ—I‘IØ×*Ñ*¨QÑ.Ø×+Ñ+Ø×5Ñ5Øô	
ˆŒ	ô ! ×!2Ñ!2Ñ3ˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r9   NÚinputsÚfinal_seq_char_positionsrz   c                 óŽ  — t        j                  |dd«      }| j                  j                  dz
  }|dz  }||z
  }t	        j
                  ||fd«      }| j                   ||«      «      }t        j                  |dd«      }| j                  |«      }| j                  |«      }| j                  |«      }|}|�t        d«      ‚|}	|	S )Nr   r›   r   z,CanineForMaskedLM is currently not supported)r4   rœ   rG   r¦   r   ÚConstantPad1dr•   r—   rQ   rU   ÚNotImplementedError)
rZ   r§   r¨   Ú	pad_totalÚpad_begÚpad_endÚpadr    Úfinal_char_seqÚ	query_seqs
             r:   r†   zConvProjection.forwardö   sÈ   € ô —‘ ¨¨AÓ.ˆð
 —K‘K×6Ñ6¸Ñ:ˆ	Ø˜q‘.ˆØ˜gÑ%ˆä×Ñ ¨Ð1°1Ó5ˆà—‘™3˜v›;Ó'ˆÜ—‘ ¨¨AÓ.ˆØ—‘ Ó(ˆØ—‘ Ó'ˆØ—‘˜fÓ%ˆØˆà#Ð/ô
 &Ð&TÓUÐUà&ˆIàÐr9   ©Nr¡   rŠ   s   @r:   r¤   r¤   ã   sD   ø„ ñô
>ð  9=ñ"à—‘ð"ð #(§,¡,°Ñ"5ð"ð 
�‰÷	"r9   r¤   c                   óÀ   ‡ — e Zd Zˆ fd„Z	 	 d	dej
                  dej
                  dej                  dz  dedz  deej
                  ej
                  dz  f   f
d„Z	ˆ xZ
S )
ÚCanineSelfAttentionc                 óÌ  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        y )Nr   rl   zThe hidden size (z6) is not a multiple of the number of attention heads (ú))rE   rF   rH   Únum_attention_headsÚhasattrrd   r‡   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluerS   Úattention_probs_dropout_probrU   r˜   s     €r:   rF   zCanineSelfAttention.__init__  s  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆ�r9   NÚfrom_tensorÚ	to_tensorÚattention_maskÚoutput_attentionsrz   c                 ó°  — |j                   \  }}}| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }	| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }
t        j                  |
|j                  dd«      «      }|t        j                  | j                  «      z  }|��|j                  dk(  rTt        j                  |d¬«      }d|j                  «       z
  t        j                  |j                   «      j"                  z  }||j%                  |j                   «      z   }t&        j(                  j+                  |d¬«      }| j-                  |«      }t        j                  ||	«      }|j/                  dddd«      j1                  «       }|j3                  «       d d | j4                  fz   } |j                  |Ž }|r||f}|S |f}|S )	NrC   r   r›   éþÿÿÿr   rn   g      ð?r   )Úshaper½   Úviewr·   r¹   rœ   r¾   r¼   r4   ÚmatmulÚmathÚsqrtÚndimÚ	unsqueezeÚfloatÚfinfor}   ÚminÚtor   Ú
functionalÚsoftmaxrU   ÚpermuteÚ
contiguousr   rº   )rZ   rÀ   rÁ   rÂ   rÃ   Ú
batch_sizerƒ   Ú_Ú	key_layerÚvalue_layerÚquery_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                   r:   r†   zCanineSelfAttention.forward.  s#  € ð %0×$5Ñ$5Ñ!ˆ
�J ð �H‰H�YÓß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�yÓ!ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�{Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%Ø×"Ñ" aÒ'ä!&§¡°ÀQÔ!G�ð #&¨×(<Ñ(<Ó(>Ñ">Ä%Ç+Á+ÐN^×NdÑNdÓBe×BiÑBiÑ!i�à/°.×2CÑ2CÐDT×DZÑDZÓ2[Ñ[Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆá6G�= /Ð2ˆàˆð O\ÐM]ˆàˆr9   ©NF)r0   r1   r2   rF   r4   r¢   r5   Úboolr7   r†   r‰   rŠ   s   @r:   r´   r´     st   ø„ ôGð, 48Ø).ñ;à—\‘\ð;ð —<‘<ð;ð ×)Ñ)¨DÑ0ð	;ð
   $™;ð;ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷;r9   r´   c                   ó’   ‡ — e Zd Zˆ fd„Zdeej                     dej                  deej                  ej                  f   fd„Zˆ xZS )ÚCanineSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr?   )rE   rF   r   r»   rH   ÚdenserQ   rR   rS   rT   rU   r˜   s     €r:   rF   zCanineSelfOutput.__init__m  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r9   r.   Úinput_tensorrz   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r²   ©rå   rU   rQ   ©rZ   r.   ræ   s      r:   r†   zCanineSelfOutput.forwards  s9   € ð Ÿ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr9   ©	r0   r1   r2   rF   r7   r4   r5   r†   r‰   rŠ   s   @r:   râ   râ   l  sL   ø„ ô>ðØ" 5×#4Ñ#4Ñ5ðØEJ×EVÑEVðà	ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4÷r9   râ   c                   óÜ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddedededededefˆ fd„Z	 	 dd
eej                     dej                  d	z  ded	z  deej                  ej                  d	z  f   fd„Z
ˆ xZS )ÚCanineAttentionav  
    Additional arguments related to local attention:

        - **local** (`bool`, *optional*, defaults to `False`) -- Whether to apply local attention.
        - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks be able to
          attend
        to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all** (`bool`,
        *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to attend to all
        positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*, defaults to 128) -- The
        width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride** (`int`, *optional*, defaults to
        128) -- The number of elements to skip when moving to the next block in `from_tensor`. -
        **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each block-wise chunk in
        *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The number of elements to
        skip when moving to the next block in `to_tensor`.
    Úalways_attend_to_first_positionÚfirst_position_attends_to_allÚattend_from_chunk_widthÚattend_from_chunk_strideÚattend_to_chunk_widthÚattend_to_chunk_stridec	                 ó  •— t         ‰	| �  «        t        |«      | _        t	        |«      | _        || _        ||k  rt        d«      ‚||k  rt        d«      ‚|| _        || _	        || _
        || _        || _        || _        y )Nze`attend_from_chunk_width` < `attend_from_chunk_stride` would cause sequence positions to get skipped.z``attend_to_chunk_width` < `attend_to_chunk_stride`would cause sequence positions to get skipped.)rE   rF   r´   rZ   râ   ÚoutputÚlocalrd   rí   rî   rï   rð   rñ   rò   ©
rZ   rG   rõ   rí   rî   rï   rð   rñ   rò   r^   s
            €r:   rF   zCanineAttention.__init__�  s˜   ø€ ô 	‰ÑÔÜ'¨Ó/ˆŒ	Ü& vÓ.ˆŒð ˆŒ
Ø"Ð%=Ò=ÜØwóð ð !Ð#9Ò9ÜØróð ð 0OˆÔ,Ø-JˆÔ*Ø'>ˆÔ$Ø(@ˆÔ%Ø%:ˆÔ"Ø&<ˆÕ#r9   Nr.   rÂ   rÃ   rz   c                 óÜ  — | j                   s| j                  ||||«      }|d   }�n|j                  d   x}}|x}}	g }
| j                  r|
j	                  d«       d}nd}t        ||| j                  «      D ].  }t        ||| j                  z   «      }|
j	                  ||f«       Œ0 g }| j                  r|j	                  d|f«       t        d|| j                  «      D ].  }t        ||| j                  z   «      }|j	                  ||f«       Œ0 t        |
«      t        |«      k7  rt        d|
› d|
› d�«      ‚g }g }t        |
|«      D ]Î  \  \  }}\  }}|d d …||…d d …f   }|	d d …||…d d …f   }|d d …||…||…f   }| j                  rN|d d …||…dd…f   }t        j                   ||gd¬«      }|	d d …dd…d d …f   }t        j                   ||gd¬«      }| j                  ||||«      }|j	                  |d   «       |sŒ»|j	                  |d   «       ŒÐ t        j                   |d¬«      }| j#                  ||«      }|f}| j                   s
|dd  z   }|S |t%        «      z   }|S )	Nr   r   )r   r   z/Expected to have same number of `from_chunks` (z) and `to_chunks` (z). Check strides.r›   rn   )rõ   rZ   rÆ   rî   re   rJ   rð   rÏ   rï   rò   rñ   rb   rd   Úziprí   r4   rr   rô   r7   )rZ   r.   rÂ   rÃ   Úself_outputsÚattention_outputÚfrom_seq_lengthÚto_seq_lengthrÀ   rÁ   Úfrom_chunksÚ
from_startÚchunk_startÚ	chunk_endÚ	to_chunksÚattention_output_chunksÚattention_probs_chunksÚfrom_endÚto_startÚto_endÚfrom_tensor_chunkÚto_tensor_chunkÚattention_mask_chunkÚcls_attention_maskÚcls_positionÚattention_outputs_chunkrÞ   s                              r:   r†   zCanineAttention.forward­  sõ  € ð �zŠzØŸ9™9 ]°MÀ>ÐSdÓeˆLØ+¨A™Òà.;×.AÑ.AÀ!Ñ.DÐDˆO˜mØ&3Ð3ˆK˜)ð ˆKØ×1Ò1Ø×"Ñ" 6Ô*ð ‘
à�
Ü$ Z°À$×B_ÑB_Ó`ò =�Ü °¸t×?[Ñ?[Ñ1[Ó\�	Ø×"Ñ" K°Ð#;Õ<ð=ð
 ˆIØ×1Ò1Ø× Ñ  ! ]Ð!3Ô4Ü$ Q¨°t×7RÑ7RÓSò ;�Ü ¨{¸T×=WÑ=WÑ/WÓX�	Ø× Ñ  +¨yÐ!9Õ:ð;ô �;Ó¤3 y£>Ò1Ü ØEÀkÀ]ð S$Ø$/ =Ð0AðCóð ð ')Ð#Ø%'Ð"Ü>AÀ+ÈyÓ>Yò NÑ:Ñ&�˜XÑ(:¨°6Ø$/²°:¸hÐ3FÊÐ0IÑ$JÐ!Ø"+ªA¨x¸¨ÂÐ,AÑ"B�ð (6²a¸ÀHÐ9LÈhÐW]ÈoÐ6]Ñ'^Ð$Ø×7Ò7Ø)7º¸:ÀhÐ;NÐPQÐRSÐPSÐ8SÑ)TÐ&Ü+0¯9©9Ð6HÐJ^Ð5_ÐefÔ+gÐ(à#,ªQ°°!°²Q¨YÑ#7�LÜ&+§i¡i°¸Ð0OÐUVÔ&W�Oà*.¯)©)Ø% Ð8LÐN_ó+Ð'ð (×.Ñ.Ð/FÀqÑ/IÔJÚ$Ø*×1Ñ1Ð2IÈ!Ñ2LÕMð%Nô(  %Ÿy™yÐ)@ÀaÔHÐàŸ;™;Ð'7¸ÓGÐØ#Ð%ˆØ�zŠzØ ¨Q¨RÐ 0Ñ0ˆGð ˆð ¤Ð&<Ó =Ñ=ˆGØˆr9   ©FFFé€   r  r  r  rß   )r0   r1   r2   r3   rà   r‡   rF   r7   r4   r5   r†   r‰   rŠ   s   @r:   rì   rì   |  sÎ   ø„ ñð& Ø05Ø.3Ø'*Ø(+Ø%(Ø&)ñ=ð *.ð	=ð
 (,ð=ð "%ð=ð #&ð=ð  #ð=ð !$õ=ðF 48Ø).ñ	Gà˜U×.Ñ.Ñ/ðGð ×)Ñ)¨DÑ0ðGð   $™;ð	Gð
 
ˆu× Ñ  %×"3Ñ"3°dÑ":Ð:Ñ	;÷Gr9   rì   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚCanineIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r²   )rE   rF   r   r»   rH   Úintermediate_sizerå   Ú
isinstancer–   Ústrr
   Úintermediate_act_fnr˜   s     €r:   rF   zCanineIntermediate.__init__ø  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r9   r.   rz   c                 óJ   — | j                  |«      }| j                  |«      }|S r²   )rå   r  ©rZ   r.   s     r:   r†   zCanineIntermediate.forward   s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr9   )r0   r1   r2   rF   r4   r5   r†   r‰   rŠ   s   @r:   r  r  ÷  s'   ø„ ô9ð U×%6Ñ%6ð ¸5×;LÑ;L÷ r9   r  c                   ót   ‡ — e Zd Zˆ fd„Zdeej                     dej                  dej                  fd„Zˆ xZS )ÚCanineOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rä   )rE   rF   r   r»   r  rH   rå   rQ   rR   rS   rT   rU   r˜   s     €r:   rF   zCanineOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r9   r.   ræ   rz   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r²   rè   ré   s      r:   r†   zCanineOutput.forward  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr9   rê   rŠ   s   @r:   r  r    s:   ø„ ô>ð U¨5×+<Ñ+<Ñ%=ð ÈU×M^ÑM^ð Ðch×ctÑct÷ r9   r  c                   ó´   ‡ — e Zd Zˆ fd„Z	 	 d	deej                     dej                  dz  dedz  deej                  ej                  dz  f   fd„Zd„ Z	ˆ xZ
S )
ÚCanineLayerc	           
      óÀ   •— t         ‰	| �  «        |j                  | _        d| _        t	        ||||||||«      | _        t        |«      | _        t        |«      | _	        y ©Nr   )
rE   rF   Úchunk_size_feed_forwardÚseq_len_dimrì   Ú	attentionr  Úintermediater  rô   rö   s
            €r:   rF   zCanineLayer.__init__  se   ø€ ô 	‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ(ØØØ+Ø)Ø#Ø$Ø!Ø"ó	
ˆŒô /¨vÓ6ˆÔÜ" 6Ó*ˆ�r9   Nr.   rÂ   rÃ   rz   c                 ó¦   — | j                  |||¬«      }|d   }|dd  }t        | j                  | j                  | j                  |«      }|f|z   }|S )N)rÃ   r   r   )r"  r   Úfeed_forward_chunkr   r!  )rZ   r.   rÂ   rÃ   Úself_attention_outputsrú   rÞ   Úlayer_outputs           r:   r†   zCanineLayer.forward0  ss   € ð "&§¡ØØØ/ð "0ó "
Ðð
 2°!Ñ4Ðà(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr9   c                 óL   — | j                  |«      }| j                  ||«      }|S r²   )r#  rô   )rZ   rú   Úintermediate_outputr'  s       r:   r%  zCanineLayer.feed_forward_chunkF  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr9   rß   )r0   r1   r2   rF   r7   r4   r5   rà   r†   r%  r‰   rŠ   s   @r:   r  r    su   ø„ ô+ð< 48Ø).ñ	à˜U×.Ñ.Ñ/ðð ×)Ñ)¨DÑ0ðð   $™;ð	ð
 
ˆu× Ñ  %×"3Ñ"3°dÑ":Ð:Ñ	;óö,r9   r  c                   ó¤   ‡ — e Zd Z	 	 	 	 	 	 	 d
ˆ fd„	Z	 	 	 	 ddeej                     dej                  dz  dedz  dedz  dedz  deez  fd	„Z	ˆ xZ
S )ÚCanineEncoderc	                 óÞ   •— t         ‰
| �  «        || _        t        j                  t        |j                  «      D �	cg c]  }	t        ||||||||«      ‘Œ c}	«      | _        d| _	        y c c}	w rß   )
rE   rF   rG   r   Ú
ModuleListrJ   Únum_hidden_layersr  ÚlayerÚgradient_checkpointing)rZ   rG   rõ   rí   rî   rï   rð   rñ   rò   rÖ   r^   s             €r:   rF   zCanineEncoder.__init__M  sx   ø€ ô 	‰ÑÔØˆŒÜ—]‘]ô ˜v×7Ñ7Ó8öð ô ØØØ3Ø1Ø+Ø,Ø)Ø*õ	òó
ˆŒ
ð ',ˆÕ#ùòs   ½A*Nr.   rÂ   rÃ   Úoutput_hidden_statesÚreturn_dictrz   c                 óô   — |rdnd }|rdnd }t        | j                  «      D ](  \  }}	|r||fz   } |	|||«      }
|
d   }|sŒ ||
d   fz   }Œ* |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )Nr8   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr²   r8   ©Ú.0Úvs     r:   ú	<genexpr>z(CanineEncoder.forward.<locals>.<genexpr>„  s   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š)r,   r.   r/   )rp   r/  r7   r   )rZ   r.   rÂ   rÃ   r1  r2  Úall_hidden_statesÚall_self_attentionsr\   Úlayer_moduleÚlayer_outputss              r:   r†   zCanineEncoder.forwardk  s¼   € ñ #7™B¸DÐÙ$5™b¸4Ðä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(¨¸ÐHYÓZˆMà)¨!Ñ,ˆMÚ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð	Pñ  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r9   r  )NFFT)r0   r1   r2   rF   r7   r4   r5   rà   r   r†   r‰   rŠ   s   @r:   r+  r+  L  s”   ø„ ð Ø(-Ø&+Ø #Ø!$Ø!Ø"õ,ðB 48Ø).Ø,1Ø#'ñ
à˜U×.Ñ.Ñ/ð
ð ×)Ñ)¨DÑ0ð
ð   $™;ð	
ð
 # T™kð
ð ˜D‘[ð
ð 
�Ñ	 ÷
r9   r+  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCaninePoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r²   )rE   rF   r   r»   rH   rå   ÚTanhr—   r˜   s     €r:   rF   zCaninePooler.__init__�  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r9   r.   rz   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rå   r—   )rZ   r.   Úfirst_token_tensorÚpooled_outputs       r:   r†   zCaninePooler.forward’  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr9   rê   rŠ   s   @r:   r?  r?  Œ  s,   ø„ ô$ð
 U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r9   r?  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCaninePredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y rä   )rE   rF   r   r»   rH   rå   r  r–   r  r
   Útransform_act_fnrQ   rR   r˜   s     €r:   rF   z&CaninePredictionHeadTransform.__init__œ  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r9   r.   rz   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r²   )rå   rH  rQ   r  s     r:   r†   z%CaninePredictionHeadTransform.forward¥  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr9   rê   rŠ   s   @r:   rF  rF  ›  s-   ø„ ôUð U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r9   rF  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCanineLMPredictionHeadc                 ó  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        t	        j                  t        j                  |j                  «      «      | _        y )NT)Úbias)rE   rF   rF  Ú	transformr   r»   rH   Ú
vocab_sizeÚdecoderÚ	Parameterr4   r€   rM  r˜   s     €r:   rF   zCanineLMPredictionHead.__init__­  s[   ø€ Ü‰ÑÔÜ6°vÓ>ˆŒô —y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒä—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	r9   r.   rz   c                 óJ   — | j                  |«      }| j                  |«      }|S r²   )rN  rP  r  s     r:   r†   zCanineLMPredictionHead.forward¹  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr9   rê   rŠ   s   @r:   rK  rK  ¬  s-   ø„ ôAð U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r9   rK  c                   ób   ‡ — e Zd Zˆ fd„Zdeej                     deej                     fd„Zˆ xZS )ÚCanineOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y r²   )rE   rF   rK  Úpredictionsr˜   s     €r:   rF   zCanineOnlyMLMHead.__init__À  s   ø€ Ü‰ÑÔÜ1°&Ó9ˆÕr9   Úsequence_outputrz   c                 ó(   — | j                  |«      }|S r²   )rV  )rZ   rW  Úprediction_scoress      r:   r†   zCanineOnlyMLMHead.forwardÄ  s   € ð !×,Ñ,¨_Ó=ÐØ Ð r9   )	r0   r1   r2   rF   r7   r4   r¢   r†   r‰   rŠ   s   @r:   rT  rT  ¿  s1   ø„ ô:ð!à˜uŸ|™|Ñ,ð!ð 
ˆu�|‰|Ñ	÷!r9   rT  c                   ó2   ‡ — e Zd ZU eed<   dZdZˆ fd„Zˆ xZS )ÚCaninePreTrainedModelrG   ÚcanineTc                 óø   •— t         ‰| �  |«       t        |t        «      rZt	        j
                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       y y )NrC   rB   )rE   Ú_init_weightsr  r<   ÚinitÚcopy_rA   r4   rW   rÆ   rY   )rZ   Úmoduler^   s     €r:   r^  z#CaninePreTrainedModel._init_weightsÒ  s[   ø€ Ü‰Ñ˜fÔ%Ü�fÔ.Ô/Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð 0r9   )	r0   r1   r2   r   r6   Úbase_model_prefixÚsupports_gradient_checkpointingr^  r‰   rŠ   s   @r:   r[  r[  Ì  s!   ø… àÓØ ÐØ&*Ð#÷ið ir9   r[  c                   óh  ‡ — e Zd Zdˆ fd„	Zd„ Zdej                  defd„Zdej                  dedej                  fd	„Z	e
	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  ded
z  ded
z  deez  fd„«       Zˆ xZS )ÚCanineModelc           
      óÒ  •— t         ‰| �  |«       || _        t        j                  |«      }d|_        t        |«      | _        t        |ddd|j                  |j                  |j                  |j                  ¬«      | _
        t        |«      | _        t        |«      | _        t        |«      | _        t        |«      | _        |rt#        |«      nd| _        | j'                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        r   TF)rõ   rí   rî   rï   rð   rñ   rò   N)rE   rF   rG   ÚcopyÚdeepcopyr.  r<   Úchar_embeddingsr+  Úlocal_transformer_strideÚinitial_char_encoderrŒ   Úchars_to_moleculesÚencoderr¤   Ú
projectionÚfinal_char_encoderr?  ÚpoolerÚ	post_init)rZ   rG   Úadd_pooling_layerÚshallow_configr^   s       €r:   rF   zCanineModel.__init__Ú  sÉ   ø€ ô
 	‰Ñ˜Ô ØˆŒÜŸ™ vÓ.ˆØ+,ˆÔ(ä/°Ó7ˆÔä$1ØØØ,1Ø*/Ø$*×$CÑ$CØ%+×%DÑ%DØ"(×"AÑ"AØ#)×#BÑ#Bô	%
ˆÔ!ô #8¸Ó"?ˆÔä$ VÓ,ˆŒÜ(¨Ó0ˆŒä"/°Ó"?ˆÔá.?”l 6Ô*ÀTˆŒð 	�‰Õr9   c                 ó  — |j                   d   |j                   d   }}|j                   d   }t        j                  ||d|f«      j                  «       }t        j                  ||dft        j
                  |j                  ¬«      }||z  }|S )aP  
        Create 3D attention mask from a 2D tensor mask.

        Args:
            from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...].
            to_mask: int32 Tensor of shape [batch_size, to_seq_length].

        Returns:
            float Tensor of shape [batch_size, from_seq_length, to_seq_length].
        r   r   )r   r}   r~   )rÆ   r4   ÚreshaperÍ   ÚonesÚfloat32r~   )rZ   rÀ   Úto_maskrÕ   rû   rü   Úbroadcast_onesÚmasks           r:   Ú)_create_3d_attention_mask_from_input_maskz5CanineModel._create_3d_attention_mask_from_input_maskü  s‹   € ð '2×&7Ñ&7¸Ñ&:¸K×<MÑ<MÈaÑ<P�Oˆ
àŸ™ aÑ(ˆä—-‘- ¨*°a¸Ð)GÓH×NÑNÓPˆô
 Ÿ™¨*°oÀqÐ)IÔQV×Q^ÑQ^Ðgn×guÑguÔvˆð  Ñ'ˆàˆr9   Úchar_attention_maskr”   c                 óÞ   — |j                   \  }}t        j                  ||d|f«      }t        j                  j	                  ||¬«      |j                  «       «      }|j                  d¬«      S )z[Downsample 2D character attention mask to 2D molecule attention mask using MaxPool1d layer.r   )r‘   r’   rn   )rÆ   r4   ru  r   Ú	MaxPool1drÍ   Úsqueeze)rZ   r|  r”   rÕ   Úchar_seq_lenÚpoolable_char_maskÚpooled_molecule_masks          r:   Ú_downsample_attention_maskz&CanineModel._downsample_attention_mask  ss   € ð $7×#<Ñ#<Ñ ˆ
�LÜ"Ÿ]™]Ð+>ÀÈQÐP\Ð@]Ó^Ðô  %Ÿx™x×1Ñ1Ð>OÐXiÐ1ÓjØ×$Ñ$Ó&ó 
Ðð
 $×+Ñ+°Ð+Ó2Ð2r9   Ú	moleculesÚchar_seq_lengthrz   c                 ó  — | j                   j                  }|dd…dd…dd…f   }t        j                  ||d¬«      }|dd…dd…dd…f   }||z  }t        j                  |||z   d¬«      }t        j                  ||gd¬«      S )zDRepeats molecules to make them the same length as the char sequence.Nr   rÅ   )Úrepeatsro   rC   rn   )rG   r”   r4   Úrepeat_interleaverr   )	rZ   r„  r…  ÚrateÚmolecules_without_extra_clsÚrepeatedÚlast_moleculeÚremainder_lengthÚremainder_repeateds	            r:   Ú_repeat_moleculeszCanineModel._repeat_molecules&  s”   € ð �{‰{×,Ñ,ˆà&/²°1±2²q°Ñ&9Ð#ä×*Ñ*Ð+FÐPTÐZ\Ô]ˆð "¢! R¡Sª! )Ñ,ˆØ*¨TÑ1ÐÜ"×4Ñ4Øà$ tÑ+Øô	
Ðô �y‰y˜(Ð$6Ð7¸RÔ@Ð@r9   Nrf   rÂ   rx   rA   ry   rÃ   r1  r2  c	                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|rdnd }
|rdnd }|�|n| j                   j                  }|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       d d }nt	        d«      ‚|\  }}|�|j                  n|j                  }|€t        j                  ||f|¬«      }|€&t        j                  |t        j                  |¬«      }| j                  || j                   j                  ¬«      }| j                  ||||¬«      }| j                  |�|n||«      }| j!                  ||||¬	«      }|j"                  }| j%                  |«      }t'        | j                   |d d …d
d…d d …f   |¬«      }| j)                  |||||¬«      }|d
   }| j*                  �| j+                  |«      nd }| j-                  ||d   ¬«      }t        j.                  ||gd¬«      }| j1                  |«      }t'        | j                   ||¬«      }| j3                  ||||¬	«      }|j"                  }|r2|r|j4                  n|d   }|
|j4                  z   |z   |j4                  z   }
|r2|r|j6                  n|d   }||j6                  z   |z   |j6                  z   }|s||f}|t9        d„ |
|fD «       «      z  }|S t;        |||
|¬«      S )Nr8   zDYou cannot specify both input_ids and inputs_embeds at the same timerC   z5You have to specify either input_ids or inputs_embeds)r~   r|   )r”   )rf   rA   rx   ry   )rÂ   rÃ   r1  r   r   )rG   ry   rÂ   )rÂ   rÃ   r1  r2  )r…  rn   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr²   r8   r5  s     r:   r8  z&CanineModel.forward.<locals>.<genexpr>Ô  s   è ø€ Òa !ÐSTÑS`œAÑaùr9  )r,   r-   r.   r/   )rG   rÃ   r1  r2  rd   Ú%warn_if_padding_and_no_attention_maskr   r~   r4   rv  r€   r�   rƒ  r”   ri  r{  rk  r,   rl  r   rm  rp  r�  rr   rn  ro  r.   r/   r7   r+   ) rZ   rf   rÂ   rx   rA   ry   rÃ   r1  r2  Úkwargsr:  r;  r‚   rÕ   rƒ   r~   Úmolecule_attention_maskÚinput_char_embeddingsr|  Úinit_chars_encoder_outputsÚinput_char_encodingÚinit_molecule_encodingÚencoder_outputsÚmolecule_sequence_outputrD  Úrepeated_moleculesÚconcatrW  Úfinal_chars_encoder_outputsÚdeep_encoder_hidden_statesÚdeep_encoder_self_attentionsrô   s                                    r:   r†   zCanineModel.forward?  s¯  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ñ #7™B¸DÐÙ$5™b¸4ÐØ%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNð #'×"AÑ"AØ¨d¯k©k×.KÑ.Kð #Bó #
Ðð
 !%× 4Ñ 4ØØ%Ø)Ø'ð	 !5ó !
Ðð #×LÑLØ"Ð.‰I°MÀ>ó
Ðð &*×%>Ñ%>Ø!Ø.Ø/Ø!5ð	 &?ó &
Ð"ð 9×JÑJÐð  "&×!8Ñ!8Ð9LÓ!MÐä";Ø—;‘;Ø0²°A°a°Cº°Ñ;Ø2ô#
Ðð Ÿ,™,Ø"Ø2Ø/Ø!5Ø#ð 'ó 
ˆð $3°1Ñ#5Ð ØAEÇÁÐAX˜Ÿ™Ð$<Ô=Ð^bˆð "×3Ñ3Ð4LÐ^iÐjlÑ^mÐ3ÓnÐô —‘Ð/Ð1CÐDÈ"ÔMˆð Ÿ/™/¨&Ó1ˆä2Ø—;‘;Ø)Ø)ô
ˆð '+×&=Ñ&=ØØ)Ø/Ø!5ð	 '>ó '
Ð#ð 6×GÑGˆáÙJU¨×)FÒ)FÐ[jÐklÑ[mÐ&à!Ø,×:Ñ:ñ;à,ñ-ð .×;Ñ;ñ<ð ñ ÙIT¨?×+EÒ+EÐZiÐjlÑZmÐ(à#Ø,×7Ñ7ñ8à.ñ/ð .×8Ñ8ñ9ð  ñ Ø% }Ð5ˆFØ”eÑaÐ(9Ð;NÐ'OÔaÓaÑaˆFØˆMä+Ø-Ø'Ø+Ø*ô	
ð 	
r9   )T)NNNNNNNN)r0   r1   r2   rF   r{  r4   r¢   r‡   rƒ  r�  r   rˆ   r5   rà   r7   r+   r†   r‰   rŠ   s   @r:   re  re  Ø  s)  ø„ õ òDð63¸e¿l¹lð 3Ð_bó 3ðA¨5¯<©<ð AÈ#ð AÐRW×R^ÑR^ó Að2 ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ñ\
à×#Ñ# dÑ*ð\
ð ×)Ñ)¨DÑ0ð\
ð ×(Ñ(¨4Ñ/ð	\
ð
 ×&Ñ&¨Ñ-ð\
ð ×(Ñ(¨4Ñ/ð\
ð   $™;ð\
ð # T™kð\
ð ˜D‘[ð\
ð 
Ð-Ñ	-ò\
ó ô\
r9   re  zž
    CANINE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
z  fd„«       Zˆ xZS )ÚCanineForSequenceClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r²   ©rE   rF   Ú
num_labelsre  r\  r   rS   rT   rU   r»   rH   Ú
classifierrq  r˜   s     €r:   rF   z(CanineForSequenceClassification.__init__æ  ói   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr9   Nrf   rÂ   rx   rA   ry   ÚlabelsrÃ   r1  r2  rz   c
           
      ó>  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|��‡| j                   j
                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j
                  dk(  r=t        «       } ||j                  d| j                  «      |j                  d«      «      }n,| j                   j
                  dk(  rt        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t!        |||j"                  |j$                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©rÂ   rx   rA   ry   rÃ   r1  r2  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrC   r›   ©ÚlossÚlogitsr.   r/   )rG   r2  r\  rU   r¥  Úproblem_typer¤  r}   r4   r�   r‡   r   r  r   rÇ   r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r§  rÃ   r1  r2  r“  rÞ   rD  r¯  r®  Úloss_fctrô   s                    r:   r†   z'CanineForSequenceClassification.forwardñ  så  € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r9   ©	NNNNNNNNN)r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r¡  r¡  ß  sû   ø„ ô	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñD
à×#Ñ# dÑ*ðD
ð ×)Ñ)¨DÑ0ðD
ð ×(Ñ(¨4Ñ/ð	D
ð
 ×&Ñ&¨Ñ-ðD
ð ×(Ñ(¨4Ñ/ðD
ð × Ñ  4Ñ'ðD
ð   $™;ðD
ð # T™kðD
ð ˜D‘[ðD
ð 
Ð)Ñ	)òD
ó ôD
r9   r¡  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
z  fd„«       Zˆ xZS )ÚCanineForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y r  )rE   rF   re  r\  r   rS   rT   rU   r»   rH   r¥  rq  r˜   s     €r:   rF   z CanineForMultipleChoice.__init__;  sV   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr9   Nrf   rÂ   rx   rA   ry   r§  rÃ   r1  r2  rz   c
           
      óJ  — |	�|	n| j                   j                  }	|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	¬«      }|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rC   rÅ   r©  r›   r­  )rG   r2  rÆ   rÇ   r   r\  rU   r¥  r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r§  rÃ   r1  r2  r“  Únum_choicesrÞ   rD  r¯  Úreshaped_logitsr®  r±  rô   s                      r:   r†   zCanineForMultipleChoice.forwardE  sÛ  € ðX &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r9   r²  )r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r´  r´  9  sû   ø„ ôð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñW
à×#Ñ# dÑ*ðW
ð ×)Ñ)¨DÑ0ðW
ð ×(Ñ(¨4Ñ/ð	W
ð
 ×&Ñ&¨Ñ-ðW
ð ×(Ñ(¨4Ñ/ðW
ð × Ñ  4Ñ'ðW
ð   $™;ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*òW
ó ôW
r9   r´  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
z  fd„«       Zˆ xZS )ÚCanineForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r²   r£  r˜   s     €r:   rF   z%CanineForTokenClassification.__init__¢  r¦  r9   Nrf   rÂ   rx   rA   ry   r§  rÃ   r1  r2  rz   c
           
      ó¦  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, CanineForTokenClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/canine-s")
        >>> model = CanineForTokenClassification.from_pretrained("google/canine-s")

        >>> inputs = tokenizer(
        ...     "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt"
        ... )

        >>> with torch.no_grad():
        ...     logits = model(**inputs).logits

        >>> predicted_token_class_ids = logits.argmax(-1)

        >>> # Note that tokens are classified rather then input words which means that
        >>> # there might be more predicted token classes than words.
        >>> # Multiple token classes might account for the same word
        >>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
        >>> predicted_tokens_classes  # doctest: +SKIP
        ```

        ```python
        >>> labels = predicted_token_class_ids
        >>> loss = model(**inputs, labels=labels).loss
        >>> round(loss.item(), 2)  # doctest: +SKIP
        ```Nr©  r   rC   r›   r­  )rG   r2  r\  rU   r¥  r   rÇ   r¤  r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r§  rÃ   r1  r2  r“  rÞ   rW  r¯  r®  r±  rô   s                    r:   r†   z$CanineForTokenClassification.forward­  sô   € ð` &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r9   r²  )r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   rº  rº     sû   ø„ ô	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñO
à×#Ñ# dÑ*ðO
ð ×)Ñ)¨DÑ0ðO
ð ×(Ñ(¨4Ñ/ð	O
ð
 ×&Ñ&¨Ñ-ðO
ð ×(Ñ(¨4Ñ/ðO
ð × Ñ  4Ñ'ðO
ð   $™;ðO
ð # T™kðO
ð ˜D‘[ðO
ð 
Ð&Ñ	&òO
ó ôO
r9   rº  c                   ó@  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚCanineForQuestionAnsweringc                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r²   )
rE   rF   r¤  re  r\  r   r»   rH   Ú
qa_outputsrq  r˜   s     €r:   rF   z#CanineForQuestionAnswering.__init__  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr9   Nrf   rÂ   rx   rA   ry   Ústart_positionsÚend_positionsrÃ   r1  r2  rz   c           
      óî  — |
�|
n| j                   j                  }
| j                  |||||||	|
¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      }|j                  d«      }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«       |j                  d|«       t        |¬«      } |||«      } |||«      }||z   dz  }|
s||f|dd  z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )	Nr©  r   r   rC   rn   )Úignore_indexr›   )r®  Ústart_logitsÚ
end_logitsr.   r/   )rG   r2  r\  rÀ  Úsplitr  rb   r   Úclamp_r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   rÁ  rÂ  rÃ   r1  r2  r“  rÞ   rW  r¯  rÅ  rÆ  Ú
total_lossÚignored_indexr±  Ú
start_lossÚend_lossrô   s                          r:   r†   z"CanineForQuestionAnswering.forward  s©  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ×"Ñ" 1 mÔ4Ø× Ñ   MÔ2ä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r9   )
NNNNNNNNNN)r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r¾  r¾     s  ø„ ôð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ñ=
à×#Ñ# dÑ*ð=
ð ×)Ñ)¨DÑ0ð=
ð ×(Ñ(¨4Ñ/ð	=
ð
 ×&Ñ&¨Ñ-ð=
ð ×(Ñ(¨4Ñ/ð=
ð ×)Ñ)¨DÑ0ð=
ð ×'Ñ'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
Ð-Ñ	-ò=
ó ô=
r9   r¾  )r´  r¾  r¡  rº  r  re  r[  )?r3   rg  rÉ   Údataclassesr   r4   r   Útorch.nnr   r   r   Ú r	   r_  Úactivationsr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   Úconfiguration_caniner   Ú
get_loggerr0   Úloggerrc   r+   ÚModuler<   rŒ   r¤   r´   râ   rì   r  r  r  r+  r?  rF  rK  rT  r[  re  r¡  r´  rº  r¾  Ú__all__r8   r9   r:   ú<module>rÜ     sG  ðñ ã Û Ý !ã Ý ß AÑ Aå &Ý !Ý 6Ý 9÷÷ õ .Ý 6ß ,Ý .ð 
ˆ×	Ñ	˜HÓ	%€ò U€ñ ðôð ô7 ;ó 7ó óð7ô:^�r—y‘yô ^ôB)˜BŸI™Iô )ôX5�R—Y‘Yô 5ôpN˜"Ÿ)™)ô Nôb�r—y‘yô ô x�b—i‘iô xôv˜Ÿ™ô ô�2—9‘9ô ô5Ð,ô 5ôp=
�B—I‘Iô =
ô@�2—9‘9ô ô B§I¡Iô ô"˜RŸY™Yô ô&
!˜Ÿ	™	ô 
!ð ôi˜Oó ió ðið ôC
Ð'ó C
ó ðC
ñL ðôôQ
Ð&;ó Q
óðQ
ðh ôc
Ð3ó c
ó ðc
ðL ô\
Ð#8ó \
ó ð\
ð~ ôI
Ð!6ó I
ó ðI
òX�r9   