Ë
    ÛÓ^j±Å  ã                   ó–  — d dl mZ d dlZd dlZd dlmZ d dlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&  e$jN                  e(«      Z) G d„ dejT                  «      Z+ G d„ dejT                  «      Z, G d„ de«      Z- G d„ de«      Z. G d„ de«      Z/ G d„ dejT                  «      Z0 G d„ dejT                  «      Z1	 	 dId ejT                  d!ejd                  d"ejd                  d#ejd                  d$ejd                  dz  d%e3dz  d&e3d'e e"   fd(„Z4 G d)„ d*ejT                  «      Z5 G d+„ d,ejT                  «      Z6 G d-„ d.e«      Z7 G d/„ d0ejT                  «      Z8 G d1„ d2ejT                  «      Z9 G d3„ d4e«      Z: G d5„ d6ejT                  «      Z;e# G d7„ d8e«      «       Z<	 	 dJd9e=e>e>f   d:e3d;e>d$ej~                  dz  d<e>d=ej€                  fd>„ZAe# G d?„ d@e<«      «       ZBdZC e#dA¬B«       G dC„ dDe<«      «       ZD e#dE¬B«       G dF„ dGe<«      «       ZEg dH¢ZFy)Ké    )ÚCallableN)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚHubertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertPositionalConvEmbeddingc                 ó  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  dz  |j                  ¬«      | _        d | _        t        |dd«      r&t        j                  |j                  «      | _        �n¯t        j                  j                  }t        t        j                  j                  d«      r$t        j                  j                  j                  }t        «       �r(dd l}|j"                  j%                  | j                  j&                  d¬«      5   || j                  dd¬	«      | _        d d d «       t        | j                  d
«      rU| j                  j                  j&                  j(                  }| j                  j                  j&                  j*                  }n,| j                  j,                  }| j                  j.                  }|j"                  j1                  | |«       |j"                  j1                  | |«       n || j                  dd¬	«      | _        t3        |j
                  «      | _        t6        |j8                     | _        y # 1 sw Y   �Œ'xY w)Né   )Úkernel_sizeÚpaddingÚgroupsÚconv_pos_batch_normFÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr!   Úhasattrr'   r   Ú	deepspeedÚzeroÚGatheredParametersr$   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr!   r5   r:   r;   Ú	__class__s         €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/hubert/modeling_hubert.pyr)   z&HubertPositionalConvEmbedding.__init__.   sÏ  ø€ Ü‰ÑÔÜ—I‘IØ×ÑØ×ÑØ×6Ñ6Ø×2Ñ2°aÑ7Ø×7Ñ7ô
ˆŒ	ð ˆŒÜ�6Ð0°%Ô8Ü Ÿn™n¨V×-?Ñ-?Ó@ˆDŽOäŸ(™(×.Ñ.ˆKÜ”r—x‘x×0Ñ0°-Ô@Ü Ÿh™h×7Ñ7×CÑC�ä)Õ+Û à—^‘^×6Ñ6°t·y±y×7GÑ7GÐWXÐ6ÓYñ MÙ +¨D¯I©I¸HÈ!Ô L�D”I÷Mä˜4Ÿ9™9Ð&8Ô9Ø#Ÿy™y×9Ñ9×@Ñ@×JÑJ�HØ#Ÿy™y×9Ñ9×@Ñ@×JÑJ‘Hà#Ÿy™y×1Ñ1�HØ#Ÿy™y×1Ñ1�HØ—‘×:Ñ:¸4ÀÔJØ—‘×:Ñ:¸4ÀÕJá'¨¯	©	¸ÀaÔH�”	ä)¨&×*HÑ*HÓIˆŒÜ  ×!?Ñ!?Ñ@ˆ�÷Mñ Mús   Å J Ê J
c                 óî   — |j                  dd«      }| j                  �| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|j                  dd«      }|S )Nr   r   )Ú	transposer0   r/   r   r?   ©r@   Úhidden_statess     rC   Úforwardz%HubertPositionalConvEmbedding.forwardS   sn   € Ø%×/Ñ/°°1Ó5ˆØ�?‰?Ð&Ø ŸO™O¨MÓ:ˆMØŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆà%×/Ñ/°°1Ó5ˆØÐó    ©Ú__name__Ú
__module__Ú__qualname__r)   rH   Ú__classcell__©rB   s   @rC   r   r   -   s   ø„ ô#AöJ	rI   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )r=   c                 óP   •— t         ‰| �  «        |dz  dk(  rd| _        y d| _        y )Nr   r   r   )r(   r)   Únum_pad_remove)r@   r-   rB   s     €rC   r)   zHubertSamePadLayer.__init__`   s)   ø€ Ü‰ÑÔØ#:¸QÑ#>À!Ò#C˜aˆÕÈˆÕrI   c                 óV   — | j                   dkD  r|d d …d d …d | j                    …f   }|S ©Nr   )rR   rF   s     rC   rH   zHubertSamePadLayer.forwardd   s6   € Ø×Ñ Ò"Ø)ª!ªQÐ0F°4×3FÑ3FÐ2FÐ0FÐ*FÑGˆMØÐrI   rJ   rO   s   @rC   r=   r=   _   s   ø„ ôKörI   r=   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertNoLayerNormConvLayerc                 ód  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        |j                     | _        y )Nr   r   ©r   ÚstrideÚbias)r(   r)   Úconv_dimÚin_conv_dimÚout_conv_dimr*   r+   Úconv_kernelÚconv_strideÚ	conv_biasr/   r   r>   r?   ©r@   rA   Úlayer_idrB   s      €rC   r)   z#HubertNoLayerNormConvLayer.__init__k   s—   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Ñ@ˆ�rI   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)r/   r?   rF   s     rC   rH   z"HubertNoLayerNormConvLayer.forwardy   s$   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØÐrI   ©r   rJ   rO   s   @rC   rV   rV   j   s   ø„ õAörI   rV   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertLayerNormConvLayerc                 ó°  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        j                  | j                  d¬«      | _        t        |j                     | _        y )Nr   r   rX   T)Úelementwise_affine)r(   r)   r[   r\   r]   r*   r+   r^   r_   r`   r/   Ú	LayerNormÚ
layer_normr   r>   r?   ra   s      €rC   r)   z!HubertLayerNormConvLayer.__init__€   s¯   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô Ÿ,™, t×'8Ñ'8ÈTÔRˆŒÜ  ×!?Ñ!?Ñ@ˆ�rI   c                 ó´   — | j                  |«      }|j                  dd«      }| j                  |«      }|j                  dd«      }| j                  |«      }|S )Néþÿÿÿéÿÿÿÿ)r/   rE   rk   r?   rF   s     rC   rH   z HubertLayerNormConvLayer.forward�   sV   € ØŸ	™	 -Ó0ˆà%×/Ñ/°°BÓ7ˆØŸ™¨Ó6ˆØ%×/Ñ/°°BÓ7ˆàŸ™¨Ó6ˆØÐrI   re   rJ   rO   s   @rC   rg   rg      s   ø„ õAörI   rg   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚHubertGroupNormConvLayerc                 óÆ  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        |j                     | _        t        j                  | j                  | j                  d¬«      | _        y )Nr   r   rX   T)Ú
num_groupsÚnum_channelsÚaffine)r(   r)   r[   r\   r]   r*   r+   r^   r_   r`   r/   r   r>   r?   Ú	GroupNormrk   ra   s      €rC   r)   z!HubertGroupNormConvLayer.__init__›   s¹   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Ñ@ˆŒäŸ,™,°$×2CÑ2CÐRV×RcÑRcÐlpÔqˆ�rI   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rd   )r/   rk   r?   rF   s     rC   rH   z HubertGroupNormConvLayer.forward«   s2   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØŸ™¨Ó6ˆØÐrI   re   rJ   rO   s   @rC   rp   rp   š   s   ø„ õrö rI   rp   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚHubertFeatureEncoderz.Construct the features from raw audio waveformc           	      óØ  •— t         ‰| �  «        |j                  dk(  rDt        |d¬«      gt	        |j
                  dz
  «      D �cg c]  }t        ||dz   ¬«      ‘Œ c}z   }nV|j                  dk(  r.t	        |j
                  «      D �cg c]  }t        ||¬«      ‘Œ }}nt        d|j                  › d�«      ‚t        j                  |«      | _        d| _        d	| _        y c c}w c c}w )
NÚgroupr   )rb   r   Úlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r(   r)   Úfeat_extract_normrp   ÚrangeÚnum_feat_extract_layersrV   rg   Ú
ValueErrorr*   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)r@   rA   Úir�   rB   s       €rC   r)   zHubertFeatureEncoder.__init__µ   sê   ø€ Ü‰ÑÔà×#Ñ# wÒ.Ü3°FÀQÔGÐHÜLQÐRX×RpÑRpÐstÑRtÓLuöLØGHÔ*¨6¸AÀ¹EÖBòLñ ‰Kð ×%Ñ%¨Ò0ÜQVÐW]×WuÑWuÓQvÖwÈAÔ3°FÀQÖGÐwˆKÑwäØ0°×1IÑ1IÐ0JÐJsÐtóð ô Ÿ=™=¨Ó5ˆÔØ&+ˆÔ#Ø"ˆÕùòLùò xs   ÁC"Â	C'c                 óJ   — | j                  «       D ]	  }d|_        Œ d| _        y ©NF)Ú
parametersÚrequires_gradrƒ   ©r@   Úparams     rC   Ú_freeze_parametersz'HubertFeatureEncoder._freeze_parametersÆ   s(   € Ø—_‘_Ó&ò 	(ˆEØ"'ˆEÕð	(à#ˆÕrI   c                 óˆ   — |d d …d f   }| j                   r| j                  rd|_        | j                  D ]
  } ||«      }Œ |S )NT)rƒ   Útrainingrˆ   r�   )r@   Úinput_valuesrG   Ú
conv_layers       rC   rH   zHubertFeatureEncoder.forwardË   sP   € Ø$¢Q¨ WÑ-ˆð ×Ò 4§=¢=Ø*.ˆMÔ'à×*Ñ*ò 	6ˆJÙ& }Ó5‰Mð	6ð ÐrI   )rK   rL   rM   Ú__doc__r)   r‹   rH   rN   rO   s   @rC   rx   rx   ²   s   ø„ Ù8ô#ò"$ö

rI   rx   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeatureProjectionc                 ón  •— t         ‰| �  «        |j                  | _        | j                  r3t        j                  |j
                  d   |j                  ¬«      | _        t        j                  |j
                  d   |j                  «      | _
        t        j                  |j                  «      | _        y )Nrn   ©Úeps)r(   r)   Úfeat_proj_layer_normr*   rj   r[   Úlayer_norm_epsrk   ÚLinearr,   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r@   rA   rB   s     €rC   r)   z HubertFeatureProjection.__init__Ù   s}   ø€ Ü‰ÑÔØ$*×$?Ñ$?ˆÔ!Ø×$Ò$Ü Ÿl™l¨6¯?©?¸2Ñ+>ÀF×DYÑDYÔZˆDŒOÜŸ)™) F§O¡O°BÑ$7¸×9KÑ9KÓLˆŒÜ—z‘z &×":Ñ":Ó;ˆ�rI   c                 ó„   — | j                   r| j                  |«      }| j                  |«      }| j                  |«      }|S rd   )r–   rk   r™   rœ   rF   s     rC   rH   zHubertFeatureProjection.forwardá   s;   € à×$Ò$Ø ŸO™O¨MÓ:ˆMØŸ™¨Ó6ˆØŸ™ ]Ó3ˆØÐrI   rJ   rO   s   @rC   r’   r’   Ø   s   ø„ ô<örI   r’   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrœ   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nrn   ç      à¿r   r   ©r&   )Úpr�   r   )
ÚsizeÚtorchÚmatmulrE   r*   Ú
functionalÚsoftmaxrœ   r�   Ú
contiguous)
rŸ   r    r¡   r¢   r£   r¤   rœ   r¥   Úattn_weightsÚattn_outputs
             rC   Úeager_attention_forwardr²   ê   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rI   c                   ó(  ‡ — e Zd ZdZ	 	 	 	 	 ddedededededed	edz  fˆ fd
„Z	 	 	 dde	j                  de	j                  dz  de	j                  dz  dedz  dee   dee	j                  e	j                  dz  ee	j                     dz  f   fd„Zˆ xZS )ÚHubertAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚ	embed_dimÚ	num_headsrœ   Ú
is_decoderrZ   Ú	is_causalrA   c                 ó
  •— t         ‰| �  «        || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _	        || _
        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r§   )rZ   )r(   r)   rµ   r¶   rœ   Úhead_dimrA   r   r¤   r·   r¸   r*   r˜   Úk_projÚv_projÚq_projÚout_proj)	r@   rµ   r¶   rœ   r·   rZ   r¸   rA   rB   s	           €rC   r)   zHubertAttention.__init__	  sä   ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒä—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�rI   rG   Úkey_value_statesr£   Úoutput_attentionsr¥   Úreturnc                 óò  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|r|n|}
g |
j                   dd ¢d‘| j                  ‘­}| j                  |
«      j                  |«      j	                  dd«      }| j                  |
«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      } || |	|||f| j                  sdn| j                  | j                  |dœ|¤Ž\  }} |j                  g |¢d‘­Ž j!                  «       }| j#                  |«      }||dfS )z#Input shape: Batch x Time x ChannelNrn   r   r   ç        )rœ   r¤   rÀ   )Úshaperº   r½   ÚviewrE   r»   r¼   r   Úget_interfacerA   Ú_attn_implementationr²   r�   rœ   r¤   Úreshaper¯   r¾   )r@   rG   r¿   r£   rÀ   r¥   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacer±   r°   s                    rC   rH   zHubertAttention.forward(  sŠ  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆá-?Ñ)À]ˆØB�^×)Ñ)¨#¨2Ð.ÐB°ÐB°D·M±MÑBˆØ—[‘[ Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆ
Ø—{‘{ >Ó2×7Ñ7¸ÓA×KÑKÈAÈqÓQˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘LØ/ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜L¨$Ð.Ð.rI   )rÃ   FTFN)NNF)rK   rL   rM   r�   ÚintÚfloatÚboolr   r)   r«   ÚTensorr   r   ÚtuplerH   rN   rO   s   @rC   r´   r´     sÿ   ø„ ÙGð Ø ØØØ&*ñCàðCð ðCð ð	Cð
 ðCð ðCð ðCð ˜tÑ#õCðD 15Ø.2Ø).ñ0/à—|‘|ð0/ð  Ÿ,™,¨Ñ-ð0/ð Ÿ™ tÑ+ð	0/ð
   $™;ð0/ð Ð-Ñ.ð0/ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷0/rI   r´   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeedForwardc                 óö  •— t         ‰| �  «        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _	        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  |j                  |j                  «      | _        t        j                  |j                   «      | _        y rd   )r(   r)   r*   rš   Úactivation_dropoutÚintermediate_dropoutr˜   r,   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutr�   s     €rC   r)   zHubertFeedForward.__init__\  s«   ø€ Ü‰ÑÔÜ$&§J¡J¨v×/HÑ/HÓ$IˆÔ!ä"$§)¡)¨F×,>Ñ,>À×@XÑ@XÓ"YˆÔÜ�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÔ$äŸI™I f×&>Ñ&>À×@RÑ@RÓSˆÔÜ Ÿj™j¨×)>Ñ)>Ó?ˆÕrI   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S rd   )rÝ   rá   rÛ   râ   rä   rF   s     rC   rH   zHubertFeedForward.forwardi  sX   € Ø×/Ñ/°Ó>ˆØ×0Ñ0°Ó?ˆØ×1Ñ1°-Ó@ˆà×)Ñ)¨-Ó8ˆØ×+Ñ+¨MÓ:ˆØÐrI   rJ   rO   s   @rC   rØ   rØ   [  s   ø„ ô@örI   rØ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚHubertEncoderLayerc                 ó°  •— t         ‰| �  «        t        |j                  |j                  |j
                  d|¬«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        y )NF©rµ   r¶   rœ   r·   rA   r”   )r(   r)   r´   r,   Únum_attention_headsÚattention_dropoutÚ	attentionr*   rš   rã   rœ   rj   r—   rk   rØ   Úfeed_forwardÚfinal_layer_normr�   s     €rC   r)   zHubertEncoderLayer.__init__t  s�   ø€ Ü‰ÑÔÜ(Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ-¨fÓ5ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÕrI   c                 óè   — |}| j                  |||¬«      \  }}}| j                  |«      }||z   }| j                  |«      }|| j                  |«      z   }| j	                  |«      }|f}|r||fz  }|S ©N©r£   rÀ   )rì   rœ   rk   rí   rî   ©r@   rG   r£   rÀ   Úattn_residualr°   Ú_Úoutputss           rC   rH   zHubertEncoderLayer.forwardƒ  s’   € Ø%ˆØ)-¯©Ø¨.ÐL]ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆàŸ™¨Ó6ˆØ%¨×(9Ñ(9¸-Ó(HÑHˆØ×-Ñ-¨mÓ<ˆà Ð"ˆáØ˜�Ñ&ˆGàˆrI   r†   rJ   rO   s   @rC   rç   rç   s  s   ø„ ô\÷rI   rç   c                   ór   ‡ — e Zd Zˆ fd„Z	 	 	 	 d	dej
                  dej                  dz  dededef
d„Zˆ xZ	S )
ÚHubertEncoderc                 ó˜  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        |«      ‘Œ c}«      | _        d| _        y c c}w ©Nr”   F)r(   r)   rA   r   Úpos_conv_embedr*   rj   r,   r—   rk   rš   rã   rœ   r€   r}   Únum_hidden_layersrç   Úlayersr‚   ©r@   rA   rô   rB   s      €rC   r)   zHubertEncoder.__init__˜  s�   ø€ Ü‰ÑÔØˆŒÜ;¸FÓCˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÌÈv×OgÑOgÓIhÖ$iÀAÔ%7¸Õ%?Ò$iÓjˆŒØ&+ˆÕ#ùò %jó   Â!CNrG   r£   rÀ   Úoutput_hidden_statesÚreturn_dictc                 óÔ  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   t        | j                  ||¬«      }| j                  |«      }	||	j                  |j                  «      z   }| j                  |«      }| j                  |«      }t        «       xs t        | «      }
| j                  D ]j  }|r||fz   }t        j                  g «      }| j                  xr || j                  j                   k  }|r|
r ||||¬«      }|d   }|rd}|sŒb|d   fz   }Œl |r||fz   }|st#        d	„ |||fD «       «      S t%        |||¬
«      S )N© rn   r   r   r   ©rA   Úinputs_embedsr£   rñ   ©NNc              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrd   r  ©Ú.0Úvs     rC   ú	<genexpr>z(HubertEncoder.forward.<locals>.<genexpr>×  ó   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š©Úlast_hidden_staterG   Ú
attentions)Ú	unsqueezeÚrepeatrÄ   r
   rA   rú   ÚtoÚdevicerk   rœ   r   r	   rü   r«   Úrandr�   Ú	layerdroprÖ   r   ©r@   rG   r£   rÀ   rÿ   r   Úall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚposition_embeddingsÚsynced_gpusr{   Údropout_probabilityÚskip_the_layerÚlayer_outputss                  rC   rH   zHubertEncoder.forward¡  s¨  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%Ð(;×(>Ñ(>¸}×?SÑ?SÓ(TÑTˆØŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6Rˆà—[‘[ò 	PˆEÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]ÒZÐ/BÀTÇ[Á[×EZÑEZÑ/ZˆNÙ!¡[á %Ø!°.ÐTeô!�ð !.¨aÑ 0�áØ ,�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð'	Pñ*  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
rI   ©NFFT)
rK   rL   rM   r)   r«   ÚtensorrÕ   rÔ   rH   rN   rO   s   @rC   r÷   r÷   —  sX   ø„ ô,ð /3Ø"'Ø%*Ø ñ;
à—|‘|ð;
ð Ÿ™ tÑ+ð;
ð  ð	;
ð
 #ð;
ð ÷;
rI   r÷   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚHubertAttnAdapterLayerc                 óœ  •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  | j
                  «      | _        t        j                  | j
                  | j                  «      | _
        t        j                  «       | _        t        j                  | j                  | j
                  «      | _        y)zŸ
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r(   r)   Úadapter_attn_dimÚ	input_dimr,   Ú
hidden_dimr*   rj   Únormr˜   Úlinear_1ÚReLUÚact_fnÚlinear_2r�   s     €rC   r)   zHubertAttnAdapterLayer.__init__à  s   ø€ ô
 	‰ÑÔØ×0Ñ0ˆŒØ ×,Ñ,ˆŒä—L‘L §¡Ó1ˆŒ	ÜŸ	™	 $§/¡/°4·>±>ÓBˆŒÜ—g‘g“iˆŒÜŸ	™	 $§.¡.°$·/±/ÓBˆ�rI   rG   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rd   )r'  r(  r*  r+  rF   s     rC   rH   zHubertAttnAdapterLayer.forwardî  s@   € ØŸ	™	 -Ó0ˆàŸ™ mÓ4ˆØŸ™ MÓ2ˆØŸ™ mÓ4ˆàÐrI   )rK   rL   rM   r)   r«   ÚFloatTensorrH   rN   rO   s   @rC   r"  r"  ß  s   ø„ ôCð U×%6Ñ%6÷ rI   r"  c                   óf   ‡ — e Zd Zˆ fd„Z	 	 ddej
                  dej
                  dz  defd„Zˆ xZS )Ú!HubertEncoderLayerStableLayerNormc                 óú  •— t         ‰| �  «        t        |j                  |j                  |j
                  d|¬«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t#        |dd «      �t%        |«      | _        y d | _        y )NFré   r”   r$  )r(   r)   r´   r,   rê   rë   rì   r*   rš   rã   rœ   rj   r—   rk   rØ   rí   rî   r1   r"  Úadapter_layerr�   s     €rC   r)   z*HubertEncoderLayerStableLayerNorm.__init__ù  sÂ   ø€ Ü‰ÑÔÜ(Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ-¨fÓ5ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔä�6Ð-¨tÓ4Ð@Ü!7¸Ó!?ˆDÕà!%ˆDÕrI   NrG   r£   rÀ   c                 ó$  — |}| j                  |«      }| j                  |||¬«      \  }}}| j                  |«      }||z   }|| j                  | j	                  |«      «      z   }| j
                  �|| j                  |«      z   }|f}|r||fz  }|S rð   )rk   rì   rœ   rí   rî   r1  rò   s           rC   rH   z)HubertEncoderLayerStableLayerNorm.forward  s±   € ð &ˆØŸ™¨Ó6ˆØ)-¯©Ø¨.ÐL]ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆØ%¨×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Ñ_ˆà×ÑÐ)Ø)¨D×,>Ñ,>¸}Ó,MÑMˆMà Ð"ˆáØ˜�Ñ&ˆGàˆrI   r†   )	rK   rL   rM   r)   r«   rÕ   rÔ   rH   rN   rO   s   @rC   r/  r/  ø  s>   ø„ ô&ð, /3Ø"'ñ	à—|‘|ðð Ÿ™ tÑ+ðð  ÷	rI   r/  c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚHubertEncoderStableLayerNormc                 ó˜  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        |«      ‘Œ c}«      | _        d| _        y c c}w rù   )r(   r)   rA   r   rú   r*   rj   r,   r—   rk   rš   rã   rœ   r€   r}   rû   r/  rü   r‚   rý   s      €rC   r)   z%HubertEncoderStableLayerNorm.__init__'  s•   ø€ Ü‰ÑÔØˆŒÜ;¸FÓCˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÜ@EÀf×F^ÑF^Ó@_Ö`¸1Ô.¨vÕ6Ò`ó
ˆŒð ',ˆÕ#ùò arþ   c                 ó¢  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   t        | j                  ||¬«      }| j                  |«      }	||	z   }| j                  |«      }t        «       xs t        | «      }
| j                  D ]j  }|r||fz   }t        j                  g «      }| j                  xr || j                  j                  k  }|r|
r ||||¬«      }|d   }|rd}|sŒb|d   fz   }Œl | j                  |«      }|r||fz   }|st        d	„ |||fD «       «      S t!        |||¬
«      S )Nr  rn   r   r   r   r  rñ   r  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrd   r  r  s     rC   r
  z7HubertEncoderStableLayerNorm.forward.<locals>.<genexpr>j  r  r  r  )r  r  rÄ   r
   rA   rú   rœ   r   r	   rü   r«   r  r�   r  rk   rÖ   r   r  s                  rC   rH   z$HubertEncoderStableLayerNorm.forward2  s™  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6Rˆà—[‘[ò 	PˆEÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]ÒZÐ/BÀTÇ[Á[×EZÑEZÑ/ZˆNÙ!¡[ñ !&Ø!°.ÐTeô!�ð !.¨aÑ 0�áØ ,�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð)	Pð, Ÿ™¨Ó6ˆáØ 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
rI   r  rJ   rO   s   @rC   r4  r4  &  s   ø„ ô	,ð ØØ"Ø÷=
rI   r4  c                   óÀ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZ ej                  «       ˆ fd„«       Zd	ej                   ez  fd
„Zdedej                   fd„Zˆ xZS )ÚHubertPreTrainedModelrA   ÚhubertrŽ   Úaudiorç   ÚParametrizedConv1dTc                 óð  •— t         ‰| �  |«       t        |t        j                  «      �rt        «       rÈddl}t        |d«      rht        |d«      r\|j                  j                  |j                  |j                  gd¬«      5  t        j                  |j                  «       ddd«       no|j                  j                  |j                  d¬«      5  t        j                  |j                  «       ddd«       nt        j                  |j                  «       |j                  � t        j                   |j                  «       yyt        |t"        «      r-t        |d«      r t        j$                  |j&                  «       yyt        |t(        «      rHt        |d«      r;t        j*                  |j,                  d| j.                  j0                  d	z   z  «       yyy# 1 sw Y   ŒÌxY w# 1 sw Y   ŒØxY w)
zInitialize the weightsr   Nr;   r:   r"   Úmasked_spec_embedÚlayer_weightsg      ð?r   )r(   Ú_init_weightsrÞ   r*   r+   r   r5   r4   r6   r7   r;   r:   ÚinitÚkaiming_normal_r$   rZ   Úzeros_ÚHubertModelÚuniform_r>  ÚHubertForSequenceClassificationÚ	constant_r?  rA   rû   )r@   rŸ   r5   rB   s      €rC   r@  z#HubertPreTrainedModel._init_weights~  s  ø€ ô 	‰Ñ˜fÔ%Ü�fœbŸi™iÕ(Ü)Ô+Û ä˜6 :Ô.´7¸6À:Ô3NØ"Ÿ™×:Ñ:¸F¿O¹OÈVÏ_É_Ð;]ÐmnÐ:Óoñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ð #Ÿ™×:Ñ:¸6¿=¹=ÐXYÐ:ÓZñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ô ×$Ñ$ V§]¡]Ô3à�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜¤Ô,Ü�vÐ2Ô3Ü—‘˜f×6Ñ6Õ7ð 4ä˜Ô ?Ô@Ü�v˜Ô/Ü—‘˜v×3Ñ3°S¸D¿K¹K×<YÑ<YÐ\]Ñ<]Ñ5^Õ_ð 0ð A÷<ð <ú÷<ð <ús   Â G Ã G,Ç G)Ç,G5Úinput_lengthsc                 ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                 ó>   — t        j                  | |z
  |d¬«      dz   S )NÚfloor)Úrounding_moder   )r«   Údiv)Úinput_lengthr   rY   s      rC   Ú_conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length�  s"   € ô —9‘9˜\¨KÑ7¸ÈwÔWÐZ[Ñ[Ð[rI   )ÚziprA   r^   r_   )r@   rH  rO  r   rY   s        rC   Ú _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengths˜  sQ   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð ÐrI   Úfeature_vector_lengthr£   c                 óä  — | j                  |j                  d«      «      j                  t        j                  «      }|j
                  d   }t        j                  ||f|j                  |j                  ¬«      }d|t        j                  |j
                  d   |j                  ¬«      |dz
  f<   |j                  dg«      j                  d«      j                  dg«      j                  «       }|S )Nrn   r   )Údtyper  r   )r  )rQ  Úsumr  r«   ÚlongrÄ   ÚzerosrT  r  ÚarangeÚflipÚcumsumrÔ   )r@   rR  r£   Úoutput_lengthsÚ
batch_sizes        rC   Ú"_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_mask§  s×   € Ø×>Ñ>¸~×?QÑ?QÐRTÓ?UÓV×YÑYÔZ_×ZdÑZdÓeˆØ#×)Ñ)¨!Ñ,ˆ
äŸ™ØÐ.Ð/°~×7KÑ7KÐTb×TiÑTiô
ˆð uvˆœŸ™ ^×%9Ñ%9¸!Ñ%<À^×EZÑEZÔ[Ð]kÐnoÑ]oÐpÑqØ'×,Ñ,¨b¨TÓ2×9Ñ9¸"Ó=×BÑBÀBÀ4ÓH×MÑMÓOˆØÐrI   )rK   rL   rM   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr«   Úno_gradr@  Ú
LongTensorrÒ   rQ  r]  rN   rO   s   @rC   r9  r9  r  s‹   ø… àÓØ ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐà€U‡]�]ƒ_ó`ó ð`ð2¸e×>NÑ>NÐQTÑ>Tó ð
Èð 
Ð]b×]mÑ]m÷ 
rI   r9  rÄ   Ú	mask_probÚmask_lengthÚ	min_masksrÁ   c                 óà  ‡‡‡‡‡— | \  }Š‰dk  rt        d«      ‚‰‰kD  rt        d‰› d‰› d�«      ‚t        j                  j                  d«      j	                  «       Šˆˆˆˆˆfd„}|�-|j                  «       j                  d«      j                  «       nt        |«      D �cg c]  }‰‘Œ c}}t        j                  |‰ft        ¬	«      }	g }
 |‰«      }|d
k(  r|	S |D ]¯  } ||«      }t        j                  j                  t        j                  |‰dz
  z
  «      |d¬«      }t        |«      d
k(  r‰dz
  }n|d
   }t        j                  |t        j                  ||z
  t        j                   ¬	«      |z  g«      }|
j#                  |«       Œ± t        j$                  |
«      }
t        j&                  |
dd…dd…df   ||‰f«      }
|
j)                  ||‰z  «      }
t        j                  ‰«      dddd…f   }t        j&                  |||‰f«      j)                  ||‰z  «      }|
|z   }
|
j+                  «       ‰dz
  kD  r‰dz
  |
|
‰dz
  kD  <   t        j,                  |	|
dd«       |	S c c}w )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 óœ   •— t        ‰| z  ‰z  ‰z   «      }t        |‰«      }|‰z  ‰kD  r‰‰z  }| ‰dz
  z
  |k  rt        | ‰dz
  z
  d«      }|S )z;Given input length, compute how many spans should be maskedr   r   )rÒ   Úmax)rN  Únum_masked_spanÚepsilonrj  ri  rk  Úsequence_lengths     €€€€€rC   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_spanÚ  so   ø€ ä˜i¨,Ñ6¸ÑDÀwÑNÓOˆÜ˜o¨yÓ9ˆð ˜[Ñ(¨?Ò:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ü! ,°+À±/Ñ"BÀAÓFˆOàÐrI   Nrn   ©rT  r   F)Úreplace)r   ÚnpÚrandomr  ÚitemÚdetachrU  Útolistr}   rW  rÔ   ÚchoicerX  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_torÈ   ro  Úput_along_axis)rÄ   ri  rj  r£   rk  r\  rs  rô   rH  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanrN  rp  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrq  rr  s    `` `            @@rC   Ú_compute_mask_indicesrŠ  ´  s­  ü€ ð0 #(Ñ€J�à�Q‚ÜÐAÓBÐBà�_Ò$ÜØ]Ð^iÐ]jØ& Ð&7°qð:ó
ð 	
ô �i‰i�n‰n˜QÓ×$Ñ$Ó&€G÷ð ð$ Ð%ð 	×ÑÓ×#Ñ# BÓ'×.Ñ.Ô0ä',¨ZÓ'8Ö9 !ŠoÒ9ð ô —H‘H˜j¨/Ð:Ä$ÔG€MØÐá1°/ÓBÐà˜aÒØÐà%ò 5ˆá1°,Ó?ˆô ŸI™I×,Ñ,Ü�I‰I�l k°A¡oÑ6Ó7¸ÐRWð -ó 
Ðô Ð Ó! QÒ&ð -¨qÑ0‰Nà.¨qÑ1ˆNäŸN™NØ¤§¡Ð(;¸oÑ(MÔUW×U]ÑU]Ô ^ÐaoÑ oÐpó
Ðð 	×!Ñ!Ð"3Õ4ð/5ô2 Ÿ™Ð"4Ó5Ðô Ÿ™Øš1ša ˜:Ñ&¨Ð5HÈ+Ð(VóÐð ,×3Ñ3°JÐ@SÐVaÑ@aÓbÐô �i‰i˜Ó$ T¨4² ]Ñ3€GÜ�o‰o˜g¨
Ð4GÈÐ'UÓV×^Ñ^ØÐ'¨+Ñ5ó€Gð ,¨gÑ5Ðð ×ÑÓ /°AÑ"5Ò5ØGVÐYZÑGZÐÐ-°À!Ñ0CÑCÑDô ×Ñ�mÐ%7¸¸BÔ?àÐùòw :s   Â$	I+c                   ó&  ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  fd„Ze		 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  deez  fd„«       Zˆ xZS )rD  rA   c                 ó²  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |j                  dkD  s|j                  dkD  rEt        j                  t        j                  |j                  «      j                  «       «      | _        |j                   rt#        |«      | _        nt'        |«      | _        | j)                  «        y ©NrÃ   )r(   r)   rA   rx   Úfeature_extractorr’   Úfeature_projectionÚmask_time_probÚmask_feature_probr*   Ú	Parameterr«   rÕ   r,   rE  r>  Údo_stable_layer_normr4  Úencoderr÷   Ú	post_initr�   s     €rC   r)   zHubertModel.__init__-  s¤   ø€ Ü‰Ñ˜Ô ØˆŒÜ!5°fÓ!=ˆÔÜ"9¸&Ó"AˆÔð × Ñ  3Ò&¨&×*BÑ*BÀSÒ*HÜ%'§\¡\´%·,±,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"à×&Ò&Ü7¸Ó?ˆD�Lä(¨Ó0ˆDŒLð 	�‰ÕrI   NrG   Úmask_time_indicesr£   c                 óÎ  — t        | j                  dd«      s|S |j                  «       \  }}}|�)| j                  j	                  |j
                  «      ||<   nË| j                  j                  dkD  r²| j                  r¦t        ||f| j                  j                  | j                  j                  || j                  j                  ¬«      }t        j                  ||j                  t        j                  ¬«      }| j                  j	                  |j
                  «      ||<   | j                  j                  dkD  r¨| j                  rœt        ||f| j                  j                  | j                  j                   | j                  j"                  ¬«      }t        j                  ||j                  t        j                  ¬«      }|dd…df   j%                  d|d«      }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )ri  rj  r£   rk  )r  rT  )ri  rj  rk  rn   )r1   rA   rª   r>  r  rT  r�  r�   rŠ  Úmask_time_lengthÚmask_time_min_masksr«   r   r  rÔ   r‘  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r@   rG   r–  r£   r\  rr  r,   Úmask_feature_indicess           rC   Ú_mask_hidden_stateszHubertModel._mask_hidden_states?  sš  € ô �t—{‘{Ð$8¸$Ô?Ø Ð ð 4A×3EÑ3EÓ3GÑ0ˆ
�O [àÐ(à/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ò,Ø�[‰[×'Ñ'¨!Ò+°·²Ü 5Ø˜_Ð-ØŸ+™+×4Ñ4Ø ŸK™K×8Ñ8Ø-ØŸ+™+×9Ñ9ô!Ðô !&§¡Ð->À}×G[ÑG[Ôch×cmÑcmÔ nÐØ/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ñ,à�;‰;×(Ñ(¨1Ò,°·²ä#8Ø˜[Ð)ØŸ+™+×7Ñ7Ø ŸK™K×;Ñ;ØŸ+™+×<Ñ<ô	$Ð ô $)§<¡<Ð0DÈ]×MaÑMaÔin×isÑisÔ#tÐ Ø#7º¸4¸Ñ#@×#GÑ#GÈÈOÐ]_Ó#`Ð Ø23ˆMÐ.Ñ/àÐrI   rŽ   rÀ   rÿ   r   rÁ   c                 óþ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      }|j                  dd«      }|�| j                  |j                  d   |«      }| j                  |«      }	| j                  |	|¬«      }	| j                  |	||||¬«      }
|
d   }	|s	|	f|
dd z   S t        |	|
j                  |
j                  ¬«      S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r   )r–  ©r£   rÀ   rÿ   r   r   r  )rA   rÀ   rÿ   r   rŽ  rE   r]  rÄ   r�  rŸ  r”  r   rG   r  )r@   rŽ   r£   r–  rÀ   rÿ   r   r¥   Úextract_featuresrG   Úencoder_outputss              rC   rH   zHubertModel.forwardm  s,  € ðJ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐàÐ%à!×DÑDÐEU×E[ÑE[Ð\]ÑE^Ð`nÓoˆNà×/Ñ/Ð0@ÓAˆØ×0Ñ0°ÐRcÐ0ÓdˆàŸ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Ñ*ˆáØ!Ð# o°a°bÐ&9Ñ9Ð9äØ+Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
rI   r  ©NNNNN)rK   rL   rM   r   r)   r«   r-  rh  rŸ  r   rÕ   rÔ   rÖ   r   rH   rN   rO   s   @rC   rD  rD  +  sõ   ø„ ð˜|õ ð* 7;Ø26ñ	,à×(Ñ(ð,ð !×,Ñ,¨tÑ3ð,ð ×(Ñ(¨4Ñ/ó	,ð\ ð /3Ø6:Ø)-Ø,0Ø#'ñE
à—l‘l TÑ)ðE
ð Ÿ™ tÑ+ðE
ð !×,Ñ,¨tÑ3ð	E
ð
   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
�Ñ	 òE
ó ôE
rI   rD  zn
    Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   óÞ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 dde	j                  dz  de	j                  dz  d	edz  d
edz  dedz  de	j                  dz  deez  fd„«       Zˆ xZS )ÚHubertForCTCNÚtarget_langc                 ó®  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        || _        |j                  €t        d| j                  › d�«      ‚t        |d«      r|j                  r|j                  n|j                  }t	        j                   ||j                  «      | _        | j%                  «        y)a0  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`HubertForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zõ with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `HubertForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r(   r)   rD  r:  r*   rš   Úfinal_dropoutrœ   r¨  Ú
vocab_sizer   rB   r4   rª  Úoutput_hidden_sizer,   r˜   Úlm_headr•  )r@   rA   r¨  r­  rB   s       €rC   r)   zHubertForCTC.__init__¿  s¿   ø€ ô 	‰Ñ˜Ô ä! &Ó)ˆŒÜ—z‘z &×"6Ñ"6Ó7ˆŒà&ˆÔà×ÑÐ$ÜØ0°·±Ð0@ð AHð Hóð ô *1°¸Ô)GÈF×L^ÒL^ˆF×%Ò%Ðdj×dvÑdvð 	ô —y‘yÐ!3°V×5FÑ5FÓGˆŒð 	�‰ÕrI   c                 ó8  — t        «       t        j                  d«      k(  ry| j                  }|�&t	        | j
                  dd«      €t        d|› d�«      ‚|€-t	        | j
                  dd«      �t        j                  d«       y|�| j                  |d¬«       yy)	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNr$  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r«   r  r¨  r1   rA   r   ÚloggerÚinfoÚload_adapter)r@   r¥   r¨  s      rC   Útie_weightszHubertForCTC.tie_weightsÜ  sœ   € ô 6Ó7¼5¿<¹<ÈÓ;OÒOØð ×&Ñ&ˆàÐ"¤w¨t¯{©{Ð<NÐPTÓ'UÐ']ÜÐ:¸;¸-ÐGtÐuÓvÐvØÐ ¤W¨T¯[©[Ð:LÈdÓ%SÐ%_Ü�K‰KÐCÕDØÐ$Ø×Ñ˜k°dÐÕ;ð %rI   c                 óL   — | j                   j                  j                  «        y©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©r:  rŽ  r‹   ©r@   s    rC   Úfreeze_feature_encoderz#HubertForCTC.freeze_feature_encoderô  ó   € ð
 	�‰×%Ñ%×8Ñ8Õ:rI   c                 óP   — | j                   j                  «       D ]	  }d|_        Œ y©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©r:  r‡   rˆ   r‰   s     rC   Úfreeze_base_modelzHubertForCTC.freeze_base_modelû  ó(   € ð
 —[‘[×+Ñ+Ó-ò 	(ˆEØ"'ˆEÕñ	(rI   rŽ   r£   rÀ   rÿ   r   ÚlabelsrÁ   c           
      ó¤  — |�|n| j                   j                  }|�I|j                  «       | j                   j                  k\  r"t	        d| j                   j                  › �«      ‚| j                  |||||¬«      }|d   }	| j                  |	«      }	| j                  |	«      }
d}|��b|�|n$t        j                  |t        j                  ¬«      }| j                  |j                  d«      «      j                  t        j                  «      }|dk\  }|j                  d«      }|j                  |«      }t        j                   j#                  |
dt        j$                  ¬«      j'                  dd«      }t        j(                  j*                  j-                  d	¬
«      5  t        j                   j/                  ||||| j                   j0                  | j                   j2                  | j                   j4                  ¬«      }ddd«       |s|
f|t6        d z   }|�|f|z   S |S t9        ||
|j:                  |j<                  ¬«      S # 1 sw Y   ŒExY w)aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: r¡  r   rt  rn   )r&   rT  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsrG   r  )rA   r   ro  r¬  r   r:  rœ   r®  r«   Ú	ones_likerV  rQ  rU  r  Úmasked_selectr*   r­   Úlog_softmaxÚfloat32rE   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityÚ_HIDDEN_STATES_START_POSITIONr   rG   r  )r@   rŽ   r£   rÀ   rÿ   r   rÁ  r¥   rõ   rG   rÉ  rÈ  rH  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probsÚoutputs                     rC   rH   zHubertForCTC.forward  s'  € ð$ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ &§*¡*£,°$·+±+×2HÑ2HÒ"HÜÐCÀDÇKÁK×DZÑDZÐC[Ð\Ó]Ð]à—+‘+ØØ)Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØŸ™ ]Ó3ˆà—‘˜mÓ,ˆàˆØÑð #1Ð"<‘Ä%Ç/Á/ÐR^Ôfk×fpÑfpÔBqð ð !×AÑAÀ.×BTÑBTÐUWÓBXÓY×\Ñ\Ô]b×]gÑ]gÓhˆMð ! A™+ˆKØ(Ÿ_™_¨RÓ0ˆNØ &× 4Ñ 4°[Ó AÐô Ÿ™×1Ñ1°&¸bÌÏÉÐ1ÓV×`Ñ`ÐabÐdeÓfˆIä—‘×%Ñ%×+Ñ+°EÐ+Ó:ñ 	Ü—}‘}×-Ñ-ØØ%Ø!Ø"ØŸ+™+×2Ñ2Ø"Ÿk™k×<Ñ<Ø"&§+¡+×"?Ñ"?ð .ó �÷	ñ Ø�Y Ô)FÐ)GÐ!HÑHˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØ˜f°G×4IÑ4IÐV]×VhÑVhô
ð 	
÷	ð 	ús   ÆA#IÉIrd   r¤  )rK   rL   rM   rà   r)   rµ  rº  r¿  r   r«   rÕ   rÔ   rÖ   r   rH   rN   rO   s   @rC   r§  r§  ¹  sÀ   ø„ ñ¨C°$©Jõ ò:<ò0;ò(ð ð /3Ø)-Ø,0Ø#'Ø&*ñE
à—l‘l TÑ)ðE
ð Ÿ™ tÑ+ðE
ð   $™;ð	E
ð
 # T™kðE
ð ˜D‘[ðE
ð —‘˜tÑ#ðE
ð 
�Ñ	òE
ó ôE
rI   r§  z•
    Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   óÊ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  d	e	dz  d
ej                  dz  de
ez  fd„«       Zˆ xZS )rF  c                 óü  •— t         ‰| �  |«       t        |d«      r|j                  rt	        d«      ‚t        |«      | _        |j                  dz   }|j                  r0t        j                  t        j                  |«      |z  «      | _        t        j                  |j                  |j                   «      | _        t        j                  |j                   |j$                  «      | _        | j)                  «        y )Nrª  z]Sequence classification does not support the use of Hubert adapters (config.add_adapter=True)r   )r(   r)   r4   rª  r   rD  r:  rû   Úuse_weighted_layer_sumr*   r’  r«   r~  r?  r˜   r,   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierr•  )r@   rA   Ú
num_layersrB   s      €rC   r)   z(HubertForSequenceClassification.__init__S  sÀ   ø€ Ü‰Ñ˜Ô ä�6˜=Ô)¨f×.@Ò.@ÜØoóð ô " &Ó)ˆŒØ×-Ñ-°Ñ1ˆ
Ø×(Ò(Ü!#§¡¬e¯j©j¸Ó.DÀzÑ.QÓ!RˆDÔÜŸ™ 6×#5Ñ#5°v×7RÑ7RÓSˆŒÜŸ)™) F×$?Ñ$?À×ARÑARÓSˆŒð 	�‰ÕrI   c                 óL   — | j                   j                  j                  «        yr·  r¸  r¹  s    rC   rº  z6HubertForSequenceClassification.freeze_feature_encoderd  r»  rI   c                 óP   — | j                   j                  «       D ]	  }d|_        Œ yr½  r¾  r‰   s     rC   r¿  z1HubertForSequenceClassification.freeze_base_modelk  rÀ  rI   NrŽ   r£   rÀ   rÿ   r   rÁ  rÁ   c                 ó<  — |�|n| j                   j                  }| j                   j                  rdn|}| j                  |||||¬«      }| j                   j                  rr|t           }	t        j                  |	d¬«      }	t        j                  j                  | j                  d¬«      }
|	|
j                  ddd«      z  j                  d¬«      }	n|d   }	| j                  |	«      }	|€|	j                  d¬«      }n‰| j                  |	j                   d   |«      }|j#                  d«      j%                  dd|	j                   d   «      }d	|	| <   |	j                  d¬«      |j                  d¬«      j                  dd«      z  }| j'                  |«      }d}|�Ft)        «       } ||j                  d| j                   j*                  «      |j                  d«      «      }|s|f|t        d z   }|�|f|z   S |S t-        |||j.                  |j0                  ¬
«      S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`HubertProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr¡  r   r¨   rn   r   r   rÃ   rÇ  )rA   r   rÝ  r:  rÕ  r«   Ústackr*   r­   r®   r?  rÅ   rU  rß  Úmeanr]  rÄ   r  r  rá  r   rà  r   rG   r  )r@   rŽ   r£   rÀ   rÿ   r   rÁ  r¥   rõ   rG   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskrÉ  rÈ  Úloss_fctrÚ  s                     rC   rH   z'HubertForSequenceClassification.forwards  s  € ð0 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ'+§{¡{×'IÒ'I™tÐOcÐà—+‘+ØØ)Ø/Ø!5Ø#ð ó 
ˆð �;‰;×-Ò-Ø#Ô$AÑBˆMÜ!ŸK™K¨¸1Ô=ˆMÜŸ=™=×0Ñ0°×1CÑ1CÈÐ0ÓLˆLØ*¨\×->Ñ->¸rÀ1ÀaÓ-HÑH×MÑMÐRSÐMÓT‰Mà# A™JˆMàŸ™ }Ó5ˆØÐ!Ø)×.Ñ.°1Ð.Ó5‰Mà×BÑBÀ=×CVÑCVÐWXÑCYÐ[iÓjˆLØ".×"8Ñ"8¸Ó"<×"CÑ"CÀAÀqÈ-×J]ÑJ]Ð^_ÑJ`Ó"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ñ-°!Ð-Ó4°|×7GÑ7GÈAÐ7GÓ7N×7SÑ7SÐTVÐXYÓ7ZÑZˆMà—‘ Ó/ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDáØ�Y Ô)FÐ)GÐ!HÑHˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rI   r¤  )rK   rL   rM   r)   rº  r¿  r   r«   rÕ   rÔ   rÖ   r   rH   rN   rO   s   @rC   rF  rF  L  s±   ø„ ôò";ò(ð ð /3Ø)-Ø,0Ø#'Ø&*ñC
à—l‘l TÑ)ðC
ð Ÿ™ tÑ+ðC
ð   $™;ð	C
ð
 # T™kðC
ð ˜D‘[ðC
ð —‘˜tÑ#ðC
ð 
Ð)Ñ	)òC
ó ôC
rI   rF  )r§  rF  rD  r9  r�  rT   )GÚcollections.abcr   Únumpyrv  r«   Útorch.nnr*   r   Ú r   rA  Úactivationsr   Úintegrations.deepspeedr   Úintegrations.fsdpr	   Úmasking_utilsr
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   r3   r   r   r   Úconfiguration_hubertr   Ú
get_loggerrK   r²  ÚModuler   r=   rV   rg   rp   rx   r’   rÕ   rÓ   r²   r´   rØ   rç   r÷   r"  r/  r4  r9  rÖ   rÒ   rh  ÚndarrayrŠ  rD  rÕ  r§  rF  Ú__all__r  rI   rC   ú<module>rÿ     sµ  ðõ* %ã Û Ý Ý %å &Ý !Ý @Ý 7Ý 6Ý BÝ 9ß YÑ Yß rÑ rÝ &ß @Ñ @Ý .ð 
ˆ×	Ñ	˜HÓ	%€ô/ B§I¡Iô /ôd˜Ÿ™ô ôÐ!;ô ô*Ð9ô ô6Ð9ô ô0#˜2Ÿ9™9ô #ôL˜bŸi™iô ð0 !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8R/�b—i‘iô R/ôj˜Ÿ	™	ô ô0!Ð3ô !ôHE
�B—I‘Iô E
ôP˜RŸY™Yô ô2+Ð(Bô +ô\I
 2§9¡9ô I
ðX ô>˜Oó >ó ð>ðJ /3ØñtØ��c�‰?ðtàðtð ðtð ×$Ñ$ tÑ+ð	tð
 ðtð ‡Z�Zótðn ôG
Ð'ó G
ó ðG
ðT !"Ð ñ ðôô
K
Ð(ó K
óð
K
ñ\ ðôôe
Ð&;ó e
óðe
òP f�rI   