Ë
    ÜÓ^jê  ã                   óF  — d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 d dlm
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+  e)jX                  e-«      Z. e(d¬«      e G d„ de«      «       «       Z/ G d„ de	j`                  «      Z1 G d„ de	j`                  «      Z2 G d„ de«      Z3 G d„ de«      Z4 G d„ d e«      Z5 G d!„ d"e	j`                  «      Z6 G d#„ d$e	j`                  «      Z7	 	 dSd%e	j`                  d&ejp                  d'ejp                  d(ejp                  d)ejp                  dz  d*e9dz  d+e9d,e%e'   fd-„Z: G d.„ d/e	j`                  «      Z; G d0„ d1e	j`                  «      Z< G d2„ d3e«      Z= G d4„ d5e	j`                  «      Z> G d6„ d7e	j`                  «      Z? G d8„ d9e«      Z@ G d:„ d;e	j`                  «      ZA G d<„ d=e	j`                  «      ZBe( G d>„ d?e"«      «       ZC	 	 dTd@eDeEeEf   dAe9dBeEd)ejŒ                  dz  dCeEdDejŽ                  fdE„ZHeZIe( G dF„ dGeC«      «       ZJ e(dH¬«       G dI„ dJeC«      «       ZKdKZL e(dL¬«       G dM„ dNeC«      «       ZM e(dO¬«       G dP„ dQeC«      «       ZNg dR¢ZOy)Ué    N)ÚCallable)Ú	dataclass)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚModelOutputÚSequenceClassifierOutputÚWav2Vec2BaseModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚUniSpeechConfigzh
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.
    )Úcustom_introc                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)	ÚUniSpeechForPreTrainingOutputaÝ  
    loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
        paper](https://huggingface.co/papers/2006.11477).
    projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
        projected quantized states.
    projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
        Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
        target vectors for contrastive loss.
    codevector_perplexity (`torch.FloatTensor` of shape `(1,)`):
        The perplexity of the codevector distribution, used to measure the diversity of the codebook.
    NÚlossÚprojected_statesÚprojected_quantized_statesÚcodevector_perplexityÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r    r!   r"   r#   Útupler$   © ó    ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/unispeech/modeling_unispeech.pyr   r   5   s”   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø15Ð�e×'Ñ'¨$Ñ.Ó5Ø;?Ð × 1Ñ 1°DÑ 8Ó?Ø6:Ð˜5×,Ñ,¨tÑ3Ó:Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r.   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechSamePadLayerc                 óP   •— t         ‰| �  «        |dz  dk(  rd| _        y d| _        y )Né   r   r   )ÚsuperÚ__init__Únum_pad_remove)ÚselfÚnum_conv_pos_embeddingsÚ	__class__s     €r/   r5   zUniSpeechSamePadLayer.__init__S   s)   ø€ Ü‰ÑÔØ#:¸QÑ#>À!Ò#C˜aˆÕÈˆÕr.   c                 óV   — | j                   dkD  r|d d …d d …d | j                    …f   }|S ©Nr   )r6   ©r7   r#   s     r/   ÚforwardzUniSpeechSamePadLayer.forwardW   s6   € Ø×Ñ Ò"Ø)ª!ªQÐ0F°4×3FÑ3FÐ2FÐ0FÐ*FÑGˆMØÐr.   ©r%   r&   r'   r5   r=   Ú__classcell__©r9   s   @r/   r1   r1   R   s   ø„ ôKör.   r1   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú UniSpeechPositionalConvEmbeddingc                 ó¦  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  dz  |j                  ¬«      | _        t        j                  j                  }t        t        j                  j                  d«      r$t        j                  j                  j                  }t        «       �r(dd l}|j                  j                  | j                  j                   d¬«      5   || j                  dd¬«      | _        d d d «       t        | j                  d«      rU| j                  j                  j                   j"                  }| j                  j                  j                   j$                  }n,| j                  j&                  }| j                  j(                  }|j                  j+                  | |«       |j                  j+                  | |«       n || j                  dd¬«      | _        t-        |j
                  «      | _        t0        |j2                     | _        y # 1 sw Y   �Œ'xY w)	Nr3   )Úkernel_sizeÚpaddingÚgroupsÚweight_normr   )Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)r4   r5   ÚnnÚConv1dÚhidden_sizer8   Únum_conv_pos_embedding_groupsÚconvÚutilsrG   ÚhasattrrL   r	   Ú	deepspeedÚzeroÚGatheredParametersrI   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterr1   rE   r   Úfeat_extract_activationÚ
activation)r7   ÚconfigrG   rT   rY   rZ   r9   s         €r/   r5   z)UniSpeechPositionalConvEmbedding.__init__^   s¥  ø€ Ü‰ÑÔÜ—I‘IØ×ÑØ×ÑØ×6Ñ6Ø×2Ñ2°aÑ7Ø×7Ñ7ô
ˆŒ	ô —h‘h×*Ñ*ˆÜ”2—8‘8×,Ñ,¨mÔ<ÜŸ(™(×3Ñ3×?Ñ?ˆKä%Õ'Ûà—‘×2Ñ2°4·9±9×3CÑ3CÐSTÐ2ÓUñ IÙ'¨¯	©	¸ÀaÔH�”	÷Iä�t—y‘yÐ"4Ô5ØŸ9™9×5Ñ5×<Ñ<×FÑF�ØŸ9™9×5Ñ5×<Ñ<×FÑF‘àŸ9™9×-Ñ-�ØŸ9™9×-Ñ-�Ø�N‰N×6Ñ6°t¸XÔFØ�N‰N×6Ñ6°t¸XÕFá# D§I¡I°HÀ!ÔDˆDŒIä,¨V×-KÑ-KÓLˆŒÜ  ×!?Ñ!?Ñ@ˆ�÷Iñ Iús   ÄIÉIc                 ó´   — |j                  dd«      }| j                  |«      }| j                  |«      }| j                  |«      }|j                  dd«      }|S )Nr   r3   )Ú	transposerQ   rE   r]   r<   s     r/   r=   z(UniSpeechPositionalConvEmbedding.forward   sV   € Ø%×/Ñ/°°1Ó5ˆàŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆà%×/Ñ/°°1Ó5ˆØÐr.   r>   r@   s   @r/   rB   rB   ]   s   ø„ ôAöBr.   rB   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechNoLayerNormConvLayerc                 ód  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        |j                     | _        y )Nr   r   ©rD   ÚstrideÚbias)r4   r5   Úconv_dimÚin_conv_dimÚout_conv_dimrM   rN   Úconv_kernelÚconv_strideÚ	conv_biasrQ   r   r\   r]   ©r7   r^   Úlayer_idr9   s      €r/   r5   z&UniSpeechNoLayerNormConvLayer.__init__‹   s—   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Ñ@ˆ�r.   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)rQ   r]   r<   s     r/   r=   z%UniSpeechNoLayerNormConvLayer.forward™   s$   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØÐr.   ©r   r>   r@   s   @r/   rb   rb   Š   s   ø„ õAör.   rb   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechLayerNormConvLayerc                 ó°  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        j                  | j                  d¬«      | _        t        |j                     | _        y )Nr   r   rd   T)Úelementwise_affine)r4   r5   rg   rh   ri   rM   rN   rj   rk   rl   rQ   Ú	LayerNormÚ
layer_normr   r\   r]   rm   s      €r/   r5   z$UniSpeechLayerNormConvLayer.__init__    s¯   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô Ÿ,™, t×'8Ñ'8ÈTÔRˆŒÜ  ×!?Ñ!?Ñ@ˆ�r.   c                 ó´   — | j                  |«      }|j                  dd«      }| j                  |«      }|j                  dd«      }| j                  |«      }|S )Néþÿÿÿéÿÿÿÿ)rQ   r`   rw   r]   r<   s     r/   r=   z#UniSpeechLayerNormConvLayer.forward¯   sV   € ØŸ	™	 -Ó0ˆà%×/Ñ/°°BÓ7ˆØŸ™¨Ó6ˆØ%×/Ñ/°°BÓ7ˆàŸ™¨Ó6ˆØÐr.   rq   r>   r@   s   @r/   rs   rs   Ÿ   s   ø„ õAör.   rs   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUniSpeechGroupNormConvLayerc                 óÆ  •— t         ‰| �  «        |dkD  r|j                  |dz
     nd| _        |j                  |   | _        t        j                  | j                  | j                  |j                  |   |j                  |   |j                  ¬«      | _
        t        |j                     | _        t        j                  | j                  | j                  d¬«      | _        y )Nr   r   rd   T)Ú
num_groupsÚnum_channelsÚaffine)r4   r5   rg   rh   ri   rM   rN   rj   rk   rl   rQ   r   r\   r]   Ú	GroupNormrw   rm   s      €r/   r5   z$UniSpeechGroupNormConvLayer.__init__»   s¹   ø€ Ü‰ÑÔØ<DÀqºL˜6Ÿ?™?¨8°a©<Ò8ÈaˆÔØ"ŸO™O¨HÑ5ˆÔä—I‘IØ×ÑØ×ÑØ×*Ñ*¨8Ñ4Ø×%Ñ% hÑ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Ñ@ˆŒäŸ,™,°$×2CÑ2CÐRV×RcÑRcÐlpÔqˆ�r.   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rp   )rQ   rw   r]   r<   s     r/   r=   z#UniSpeechGroupNormConvLayer.forwardË   s2   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØŸ™¨Ó6ˆØÐr.   rq   r>   r@   s   @r/   r|   r|   º   s   ø„ õrö r.   r|   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )ÚUniSpeechFeatureEncoderz.Construct the features from raw audio waveformc           	      óØ  •— t         ‰| �  «        |j                  dk(  rDt        |d¬«      gt	        |j
                  dz
  «      D �cg c]  }t        ||dz   ¬«      ‘Œ c}z   }nV|j                  dk(  r.t	        |j
                  «      D �cg c]  }t        ||¬«      ‘Œ }}nt        d|j                  › d�«      ‚t        j                  |«      | _        d| _        d	| _        y c c}w c c}w )
NÚgroupr   )rn   r   Úlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r4   r5   Úfeat_extract_normr|   ÚrangeÚnum_feat_extract_layersrb   rs   Ú
ValueErrorrM   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)r7   r^   Úir�   r9   s       €r/   r5   z UniSpeechFeatureEncoder.__init__Õ   sñ   ø€ Ü‰ÑÔà×#Ñ# wÒ.Ü6°vÈÔJÐKä˜v×=Ñ=ÀÑAÓBöOàô .¨f¸qÀ1¹uÖEòOñ ‰Kð ×%Ñ%¨Ò0äINÈv×OmÑOmÓInöØDEÔ+¨F¸QÖ?ðˆKñ ô Ø0°×1IÑ1IÐ0JÐJsÐtóð ô Ÿ=™=¨Ó5ˆÔØ&+ˆÔ#Ø"ˆÕùòOùò
s   ÁC"Â	C'c                 óJ   — | j                  «       D ]	  }d|_        Œ d| _        y ©NF)Ú
parametersÚrequires_gradr�   ©r7   Úparams     r/   Ú_freeze_parametersz*UniSpeechFeatureEncoder._freeze_parametersé   s(   € Ø—_‘_Ó&ò 	(ˆEØ"'ˆEÕð	(à#ˆÕr.   c                 óˆ   — |d d …d f   }| j                   r| j                  rd|_        | j                  D ]
  } ||«      }Œ |S )NT)r�   Útrainingr”   r�   )r7   Úinput_valuesr#   Ú
conv_layers       r/   r=   zUniSpeechFeatureEncoder.forwardî   sP   € Ø$¢Q¨ WÑ-ˆð ×Ò 4§=¢=Ø*.ˆMÔ'à×*Ñ*ò 	6ˆJÙ& }Ó5‰Mð	6ð Ðr.   )r%   r&   r'   r(   r5   r—   r=   r?   r@   s   @r/   r„   r„   Ò   s   ø„ Ù8ô#ò($ö

r.   r„   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechFeatureProjectionc                 ó4  •— t         ‰| �  «        t        j                  |j                  d   |j
                  ¬«      | _        t        j                  |j                  d   |j                  «      | _	        t        j                  |j                  «      | _        y )Nrz   ©Úeps)r4   r5   rM   rv   rg   Úlayer_norm_epsrw   ÚLinearrO   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r7   r^   r9   s     €r/   r5   z#UniSpeechFeatureProjection.__init__ü   sf   ø€ Ü‰ÑÔÜŸ,™, v§¡°rÑ':À×@UÑ@UÔVˆŒÜŸ)™) F§O¡O°BÑ$7¸×9KÑ9KÓLˆŒÜ—z‘z &×":Ñ":Ó;ˆ�r.   c                 óp   — | j                  |«      }| j                  |«      }| j                  |«      }||fS rp   )rw   r£   r¦   )r7   r#   Únorm_hidden_statess      r/   r=   z"UniSpeechFeatureProjection.forward  s:   € à!Ÿ_™_¨]Ó;ÐØŸ™Ð(:Ó;ˆØŸ™ ]Ó3ˆØÐ0Ð0Ð0r.   r>   r@   s   @r/   r�   r�   û   s   ø„ ô<ö1r.   r�   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr¦   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nrz   ç      à¿r3   r   ©rK   )Úpr™   r   )
Úsizer)   Úmatmulr`   rM   Ú
functionalÚsoftmaxr¦   r™   Ú
contiguous)
rª   r«   r¬   r­   r®   r¯   r¦   r°   Úattn_weightsÚattn_outputs
             r/   Úeager_attention_forwardr¼   
  s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r.   c                   ó(  ‡ — e Zd ZdZ	 	 	 	 	 ddedededededed	edz  fˆ fd
„Z	 	 	 dde	j                  de	j                  dz  de	j                  dz  dedz  dee   dee	j                  e	j                  dz  ee	j                     dz  f   fd„Zˆ xZS )ÚUniSpeechAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚ	embed_dimÚ	num_headsr¦   Ú
is_decoderrf   Ú	is_causalr^   c                 ó
  •— t         ‰| �  «        || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _	        || _
        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r²   )rf   )r4   r5   r¿   rÀ   r¦   Úhead_dimr^   r‹   r¯   rÁ   rÂ   rM   r¢   Úk_projÚv_projÚq_projÚout_proj)	r7   r¿   rÀ   r¦   rÁ   rf   rÂ   r^   r9   s	           €r/   r5   zUniSpeechAttention.__init__)  sä   ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒä—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�r.   r#   Úkey_value_statesr®   Úoutput_attentionsr°   Úreturnc                 óò  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|r|n|}
g |
j                   dd ¢d‘| j                  ‘­}| j                  |
«      j                  |«      j	                  dd«      }| j                  |
«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      } || |	|||f| j                  sdn| j                  | j                  |dœ|¤Ž\  }} |j                  g |¢d‘­Ž j!                  «       }| j#                  |«      }||dfS )z#Input shape: Batch x Time x ChannelNrz   r   r3   ç        )r¦   r¯   rÊ   )ÚshaperÄ   rÇ   Úviewr`   rÅ   rÆ   r   Úget_interfacer^   Ú_attn_implementationr¼   r™   r¦   r¯   Úreshaper¹   rÈ   )r7   r#   rÉ   r®   rÊ   r°   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacer»   rº   s                    r/   r=   zUniSpeechAttention.forwardH  sŠ  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆá-?Ñ)À]ˆØB�^×)Ñ)¨#¨2Ð.ÐB°ÐB°D·M±MÑBˆØ—[‘[ Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆ
Ø—{‘{ >Ó2×7Ñ7¸ÓA×KÑKÈAÈqÓQˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘LØ/ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜L¨$Ð.Ð.r.   )rÍ   FTFN)NNF)r%   r&   r'   r(   ÚintÚfloatÚboolr   r5   r)   ÚTensorr   r   r,   r=   r?   r@   s   @r/   r¾   r¾   &  sÿ   ø„ ÙGð Ø ØØØ)-ñCàðCð ðCð ð	Cð
 ðCð ðCð ðCð   $Ñ&õCðD 15Ø.2Ø).ñ0/à—|‘|ð0/ð  Ÿ,™,¨Ñ-ð0/ð Ÿ™ tÑ+ð	0/ð
   $™;ð0/ð Ð-Ñ.ð0/ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷0/r.   r¾   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚUniSpeechFeedForwardc                 óö  •— t         ‰| �  «        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _	        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  |j                  |j                  «      | _        t        j                  |j                   «      | _        y rp   )r4   r5   rM   r¤   Úactivation_dropoutÚintermediate_dropoutr¢   rO   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutr§   s     €r/   r5   zUniSpeechFeedForward.__init__|  s«   ø€ Ü‰ÑÔÜ$&§J¡J¨v×/HÑ/HÓ$IˆÔ!ä"$§)¡)¨F×,>Ñ,>À×@XÑ@XÓ"YˆÔÜ�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÔ$äŸI™I f×&>Ñ&>À×@RÑ@RÓSˆÔÜ Ÿj™j¨×)>Ñ)>Ó?ˆÕr.   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S rp   )ræ   rê   rä   rë   rí   r<   s     r/   r=   zUniSpeechFeedForward.forward‰  sX   € Ø×/Ñ/°Ó>ˆØ×0Ñ0°Ó?ˆØ×1Ñ1°-Ó@ˆà×)Ñ)¨-Ó8ˆØ×+Ñ+¨MÓ:ˆØÐr.   r>   r@   s   @r/   rá   rá   {  s   ø„ ô@ör.   rá   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚUniSpeechEncoderLayerc                 ó°  •— t         ‰| �  «        t        |j                  |j                  |j
                  d|¬«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        y )NF©r¿   rÀ   r¦   rÁ   r^   rŸ   )r4   r5   r¾   rO   Únum_attention_headsÚattention_dropoutÚ	attentionrM   r¤   rì   r¦   rv   r¡   rw   rá   Úfeed_forwardÚfinal_layer_normr§   s     €r/   r5   zUniSpeechEncoderLayer.__init__”  s�   ø€ Ü‰ÑÔÜ+Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ0°Ó8ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÕr.   c                 óè   — |}| j                  |||¬«      \  }}}| j                  |«      }||z   }| j                  |«      }|| j                  |«      z   }| j	                  |«      }|f}|r||fz  }|S ©N©r®   rÊ   )rõ   r¦   rw   rö   r÷   ©r7   r#   r®   rÊ   Úattn_residualrº   Ú_Úoutputss           r/   r=   zUniSpeechEncoderLayer.forward£  s’   € Ø%ˆØ)-¯©Ø¨.ÐL]ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆàŸ™¨Ó6ˆØ%¨×(9Ñ(9¸-Ó(HÑHˆØ×-Ñ-¨mÓ<ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr.   r’   r>   r@   s   @r/   rð   rð   “  s   ø„ ô\÷r.   rð   c                   ór   ‡ — e Zd Zˆ fd„Z	 	 	 	 d	dej
                  dej                  dz  dededef
d„Zˆ xZ	S )
ÚUniSpeechEncoderc                 ó˜  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        |«      ‘Œ c}«      | _        d| _        y c c}w ©NrŸ   F)r4   r5   r^   rB   Úpos_conv_embedrM   rv   rO   r¡   rw   r¤   rì   r¦   rŒ   r‰   Únum_hidden_layersrð   ÚlayersrŽ   ©r7   r^   rý   r9   s      €r/   r5   zUniSpeechEncoder.__init__¸  s‘   ø€ Ü‰ÑÔØˆŒÜ>¸vÓFˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÌEÐRX×RjÑRjÓLkÖ$lÀqÔ%:¸6Õ%BÒ$lÓmˆŒØ&+ˆÕ#ùò %mó   Â!CNr#   r®   rÊ   Úoutput_hidden_statesÚreturn_dictc                 óÔ  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   t        | j                  ||¬«      }| j                  |«      }	||	j                  |j                  «      z   }| j                  |«      }| j                  |«      }t        «       xs t        | «      }
| j                  D ]j  }|r||fz   }t        j                  g «      }| j                  xr || j                  j                   k  }|r|
r ||||¬«      }|d   }|rd}|sŒb|d   fz   }Œl |r||fz   }|st#        d	„ |||fD «       «      S t%        |||¬
«      S )Nr-   rz   r   r3   r   ©r^   Úinputs_embedsr®   rú   ©NNc              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrp   r-   ©Ú.0Úvs     r/   ú	<genexpr>z+UniSpeechEncoder.forward.<locals>.<genexpr>÷  ó   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š©Úlast_hidden_stater#   r$   )Ú	unsqueezeÚrepeatrÎ   r   r^   r  ÚtoÚdevicerw   r¦   r	   r
   r  r)   Úrandr™   Ú	layerdropr,   r   ©r7   r#   r®   rÊ   r  r	  Úall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚposition_embeddingsÚsynced_gpusr‡   Údropout_probabilityÚskip_the_layerÚlayer_outputss                  r/   r=   zUniSpeechEncoder.forwardÁ  s¨  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%Ð(;×(>Ñ(>¸}×?SÑ?SÓ(TÑTˆØŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6Rˆà—[‘[ò 	PˆEÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]ÒZÐ/BÀTÇ[Á[×EZÑEZÑ/ZˆNÙ!¡[á %Ø!°.ÐTeô!�ð !.¨aÑ 0�áØ ,�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð'	Pñ*  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r.   ©NFFT)
r%   r&   r'   r5   r)   Útensorrß   rÞ   r=   r?   r@   s   @r/   r   r   ·  sX   ø„ ô,ð /3Ø"'Ø%*Ø ñ;
à—|‘|ð;
ð Ÿ™ tÑ+ð;
ð  ð	;
ð
 #ð;
ð ÷;
r.   r   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚUniSpeechAttnAdapterLayerc                 óœ  •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  | j
                  «      | _        t        j                  | j
                  | j                  «      | _
        t        j                  «       | _        t        j                  | j                  | j
                  «      | _        y)zŸ
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r4   r5   Úadapter_attn_dimÚ	input_dimrO   Ú
hidden_dimrM   rv   Únormr¢   Úlinear_1ÚReLUÚact_fnÚlinear_2r§   s     €r/   r5   z"UniSpeechAttnAdapterLayer.__init__   s   ø€ ô
 	‰ÑÔØ×0Ñ0ˆŒØ ×,Ñ,ˆŒä—L‘L §¡Ó1ˆŒ	ÜŸ	™	 $§/¡/°4·>±>ÓBˆŒÜ—g‘g“iˆŒÜŸ	™	 $§.¡.°$·/±/ÓBˆ�r.   r#   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rp   )r.  r/  r1  r2  r<   s     r/   r=   z!UniSpeechAttnAdapterLayer.forward  s@   € ØŸ	™	 -Ó0ˆàŸ™ mÓ4ˆØŸ™ MÓ2ˆØŸ™ mÓ4ˆàÐr.   )r%   r&   r'   r5   r)   r*   r=   r?   r@   s   @r/   r)  r)  ÿ  s   ø„ ôCð U×%6Ñ%6÷ r.   r)  c                   óf   ‡ — e Zd Zˆ fd„Z	 	 ddej
                  dej
                  dz  defd„Zˆ xZS )Ú$UniSpeechEncoderLayerStableLayerNormc                 óú  •— t         ‰| �  «        t        |j                  |j                  |j
                  d|¬«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t#        |dd «      �t%        |«      | _        y d | _        y )NFrò   rŸ   r+  )r4   r5   r¾   rO   ró   rô   rõ   rM   r¤   rì   r¦   rv   r¡   rw   rá   rö   r÷   Úgetattrr)  Úadapter_layerr§   s     €r/   r5   z-UniSpeechEncoderLayerStableLayerNorm.__init__  sÂ   ø€ Ü‰ÑÔÜ+Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ0°Ó8ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔä�6Ð-¨tÓ4Ð@Ü!:¸6Ó!BˆDÕà!%ˆDÕr.   Nr#   r®   rÊ   c                 ó$  — |}| j                  |«      }| j                  |||¬«      \  }}}| j                  |«      }||z   }|| j                  | j	                  |«      «      z   }| j
                  �|| j                  |«      z   }|f}|r||fz  }|S rù   )rw   rõ   r¦   rö   r÷   r8  rû   s           r/   r=   z,UniSpeechEncoderLayerStableLayerNorm.forward,  s±   € ð &ˆØŸ™¨Ó6ˆØ)-¯©Ø¨.ÐL]ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆØ%¨×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Ñ_ˆà×ÑÐ)Ø)¨D×,>Ñ,>¸}Ó,MÑMˆMà Ð"ˆáØ˜�Ñ&ˆGàˆr.   r’   )	r%   r&   r'   r5   r)   rß   rÞ   r=   r?   r@   s   @r/   r5  r5    s>   ø„ ô&ð, /3Ø"'ñ	à—|‘|ðð Ÿ™ tÑ+ðð  ÷	r.   r5  c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚUniSpeechEncoderStableLayerNormc                 ó˜  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        |«      ‘Œ c}«      | _        d| _        y c c}w r  )r4   r5   r^   rB   r  rM   rv   rO   r¡   rw   r¤   rì   r¦   rŒ   r‰   r  r5  r  rŽ   r  s      €r/   r5   z(UniSpeechEncoderStableLayerNorm.__init__G  s•   ø€ Ü‰ÑÔØˆŒÜ>¸vÓFˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÜCHÈ×IaÑIaÓCbÖc¸aÔ1°&Õ9Òcó
ˆŒð ',ˆÕ#ùò dr  c                 ó¢  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   t        | j                  ||¬«      }| j                  |«      }	||	z   }| j                  |«      }t        «       xs t        | «      }
| j                  D ]j  }|r||fz   }t        j                  g «      }| j                  xr || j                  j                  k  }|r|
r ||||¬«      }|d   }|rd}|sŒb|d   fz   }Œl | j                  |«      }|r||fz   }|st        d	„ |||fD «       «      S t!        |||¬
«      S )Nr-   rz   r   r3   r   r  rú   r  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrp   r-   r  s     r/   r  z:UniSpeechEncoderStableLayerNorm.forward.<locals>.<genexpr>Š  r  r  r  )r  r  rÎ   r   r^   r  r¦   r	   r
   r  r)   r  r™   r  rw   r,   r   r  s                  r/   r=   z'UniSpeechEncoderStableLayerNorm.forwardR  s™  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6Rˆà—[‘[ò 	PˆEÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]ÒZÐ/BÀTÇ[Á[×EZÑEZÑ/ZˆNÙ!¡[ñ !&Ø!°.ÐTeô!�ð !.¨aÑ 0�áØ ,�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð)	Pð, Ÿ™¨Ó6ˆáØ 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r.   r&  r>   r@   s   @r/   r;  r;  F  s   ø„ ô	,ð ØØ"Ø÷=
r.   r;  c                   ó8   ‡ — e Zd ZdZˆ fd„Zed„ «       Zd„ Zˆ xZS )ÚUniSpeechGumbelVectorQuantizerz±
    Vector quantization using gumbel softmax. See `[CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information.
    c                 ó0  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | j                  z  dk7  r&t        d|j                  › d| j                  › d�«      ‚t        j                  t        j                  d| j                  | j
                  z  |j                  | j                  z  «      «      | _        t        j                  |j                  d   | j                  | j
                  z  «      | _        d| _        y )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenationr   rz   r3   )r4   r5   Únum_codevector_groupsr~   Únum_codevectors_per_groupÚnum_varsÚcodevector_dimr‹   rM   Ú	Parameterr)   r*   Úcodevectorsr¢   rg   Úweight_projÚtemperaturer§   s     €r/   r5   z'UniSpeechGumbelVectorQuantizer.__init__˜  sì   ø€ Ü‰ÑÔØ ×6Ñ6ˆŒØ×8Ñ8ˆŒà× Ñ  4§?¡?Ñ2°aÒ7ÜØ)¨&×*?Ñ*?Ð)@ð A5Ø59·_±_Ð4EÐEWðYóð ô Ÿ<™<Ü×Ñ˜a §¡°4·=±=Ñ!@À&×BWÑBWÐ[_×[jÑ[jÑBjÓkó
ˆÔô Ÿ9™9 V§_¡_°RÑ%8¸$¿/¹/ÈDÏMÉMÑ:YÓZˆÔð ˆÕr.   c                 óÄ   — | j                  d¬«      }t        j                  t        j                  t        j                  ||«      d¬«       «      j                  «       }|S )Nr   r³   rz   )Úmeanr)   ÚexpÚsumÚxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r/   Ú_compute_perplexityz2UniSpeechGumbelVectorQuantizer._compute_perplexity¬  sI   € àŸ™¨˜Ó*ˆÜ—Y‘Y¤§	¡	¬%¯+©+°nÀnÓ*UÐ[]Ô ^Ð^Ó_×cÑcÓeˆ
ØÐr.   c                 óÞ  — |j                   \  }}}| j                  |«      }|j                  ||z  | j                  z  d«      }| j                  ržt
        j                  j                  |j                  «       | j                  d¬«      j                  |«      }t        j                  |j                  ||z  | j                  d«      j                  «       d¬«      }| j                  |«      }n}|j                  d¬«      } |j                  |j                   Ž j!                  d|j                  dd«      d«      }|j                  ||z  | j                  d«      }| j                  |«      }|j                  ||z  d«      }|j#                  d«      | j$                  z  }	|	j                  ||z  | j                  | j&                  d«      }
|
j)                  d«      j                  ||d«      }
|
|fS )Nrz   T)ÚtauÚhardr³   r   g      ð?ry   )rÎ   rH  rÏ   r~   r™   rM   r·   Úgumbel_softmaxrÝ   rI  Útype_asr)   r¸   rR  ÚargmaxÚ	new_zerosÚscatter_r  rG  rD  rM  )r7   r#   Ú
batch_sizeÚsequence_lengthrO   Úcodevector_probsÚcodevector_soft_distrQ  Úcodevector_idxÚcodevectors_per_grouprG  s              r/   r=   z&UniSpeechGumbelVectorQuantizer.forward²  sÛ  € Ø3@×3FÑ3FÑ0ˆ
�O [ð ×(Ñ(¨Ó7ˆØ%×*Ñ*¨:¸Ñ+GÈ$Ï/É/Ñ+YÐ[]Ó^ˆà�=Š=ä!Ÿ}™}×;Ñ;Ø×#Ñ#Ó%¨4×+;Ñ+;À$ð  <ó  ç‰g�mÓ$ð ô
 $)§=¡=Ø×"Ñ" :°Ñ#?ÀÇÁÐRTÓU×[Ñ[Ó]Ðceô$Ð ð ×1Ñ1Ð2FÓG‰Jð +×1Ñ1°bÐ1Ó9ˆNØ6˜}×6Ñ6¸×8KÑ8KÐL×UÑUØ�N×'Ñ'¨¨AÓ.°ó Ðð  0×4Ñ4°ZÀ/Ñ5QÐSW×SbÑSbÐdfÓgÐà×1Ñ1Ð2BÓCˆJà+×0Ñ0°¸oÑ1MÈrÓRÐà 0× :Ñ :¸2Ó >À×AQÑAQÑ QÐØ+×0Ñ0°¸oÑ1MÈtÏÉÐ`d×`mÑ`mÐoqÓrˆØ!—o‘o bÓ)×.Ñ.¨z¸?ÈBÓOˆà˜JÐ&Ð&r.   )	r%   r&   r'   r(   r5   ÚstaticmethodrR  r=   r?   r@   s   @r/   r@  r@  ’  s&   ø„ ñô
ð( ñó ðö
#'r.   r@  c                   ó¸   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej                  «       ˆ fd„«       Zdej                  ez  fd„Zd	ed
ej                  fd„Zˆ xZS )ÚUniSpeechPreTrainedModelr^   Ú	unispeechrš   ÚaudioTc           
      ó  •— t         ‰| �  |«       t        |t        «      rut	        j
                  |j                  j                  dd¬«       t	        j                  |j                  j                  «       t	        j                  |j                  «       yt        |t        «      ržt	        j
                  |j                  j                  ddt        j                  d|j                  j                   d   |j                  j"                  z  z  «      z  ¬«       t	        j$                  |j                  j                  d«       yt        |t&        «      r‡t        j                  d|j(                  j*                  z  «      }t	        j                  |j(                  j                  | |¬«       t	        j                  |j(                  j                  | |¬«       yt        |t,        j.                  «      rŒt	        j0                  |j                  «       |j                  �`t        j                  |j2                  |j"                  |j                   d   z  z  «      }t	        j                  |j                  | |¬«       yyy)zInitialize the weightsrÍ   r   )rK  Ústdr   r3   )ÚaÚbN)r4   Ú_init_weightsrç   r@  ÚinitÚnormal_rH  rI   Úzeros_rf   Úuniform_rG  rB   rQ   ÚmathÚsqrtrD   Úin_channelsÚ	constant_r�   r£   Úin_featuresrM   rN   Úkaiming_normal_rF   )r7   rª   Úkr9   s      €r/   rj  z&UniSpeechPreTrainedModel._init_weightsã  s¶  ø€ ô 	‰Ñ˜fÔ%ä�fÔ<Ô=Ü�L‰L˜×+Ñ+×2Ñ2¸À!ÕDÜ�K‰K˜×*Ñ*×/Ñ/Ô0Ü�M‰M˜&×,Ñ,Õ-Ü˜Ô @ÔAÜ�L‰LØ—‘×"Ñ"ØØœŸ	™	 ! v§{¡{×'>Ñ'>¸qÑ'AÀFÇKÁK×D[ÑD[Ñ'[Ñ"\Ó]Ñ]õô
 �N‰N˜6Ÿ;™;×+Ñ+¨QÕ/Ü˜Ô :Ô;Ü—	‘	˜!˜f×/Ñ/×;Ñ;Ñ;Ó<ˆAÜ�M‰M˜&×+Ñ+×2Ñ2°q°b¸AÕ>Ü�M‰M˜&×+Ñ+×0Ñ0°Q°B¸!Ö<Ü˜¤§	¡	Ô*Ü× Ñ  §¡Ô/à�{‰{Ð&Ü—I‘I˜fŸm™m¨v×/AÑ/AÀF×DVÑDVÐWXÑDYÑ/YÑZÓ[�Ü—‘˜fŸk™k¨a¨R°1Ö5ð 'ð +r.   Úinput_lengthsc                 ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                 ó>   — t        j                  | |z
  |d¬«      dz   S )NÚfloor)Úrounding_moder   )r)   Údiv)Úinput_lengthrD   re   s      r/   Ú_conv_out_lengthzSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length  s"   € ô —9‘9˜\¨KÑ7¸ÈwÔWÐZ[Ñ[Ð[r.   )Úzipr^   rj   rk   )r7   rv  r}  rD   re   s        r/   Ú _get_feat_extract_output_lengthsz9UniSpeechPreTrainedModel._get_feat_extract_output_lengthsþ  sQ   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð Ðr.   Úfeature_vector_lengthr®   c                 óø  — |j                  d¬«      d d …df   }| j                  |«      j                  t        j                  «      }|j
                  d   }t        j                  ||f|j                  |j                  ¬«      }d|t        j                  |j
                  d   |j                  ¬«      |dz
  f<   |j                  dg«      j                  d«      j                  dg«      j                  «       }|S )Nrz   r³   r   )Údtyper  r   )r  )Úcumsumr  r  r)   ÚlongrÎ   Úzerosr‚  r  ÚarangeÚfliprÞ   )r7   r€  r®   Únon_padded_lengthsÚoutput_lengthsr[  s         r/   Ú"_get_feature_vector_attention_maskz;UniSpeechPreTrainedModel._get_feature_vector_attention_mask  së   € ð ,×2Ñ2°rÐ2Ó:º1¸b¸5ÑAÐØ×>Ñ>Ð?QÓR×UÑUÔV[×V`ÑV`ÓaˆØ#×)Ñ)¨!Ñ,ˆ
äŸ™ØÐ.Ð/°~×7KÑ7KÐTb×TiÑTiô
ˆð uvˆœŸ™ ^×%9Ñ%9¸!Ñ%<À^×EZÑEZÔ[Ð]kÐnoÑ]oÐpÑqØ'×,Ñ,¨b¨TÓ2×9Ñ9¸"Ó=×BÑBÀBÀ4ÓH×MÑMÓOˆØÐr.   )r%   r&   r'   r   r+   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr)   Úno_gradrj  Ú
LongTensorrÜ   r  rŠ  r?   r@   s   @r/   rc  rc  Ø  s}   ø… àÓØ#ÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà€U‡]�]ƒ_ó6ó ð6ð4¸e×>NÑ>NÐQTÑ>Tó ðÈð Ð]b×]mÑ]m÷ r.   rc  rÎ   Ú	mask_probÚmask_lengthÚ	min_masksrË   c                 óà  ‡‡‡‡‡— | \  }Š‰dk  rt        d«      ‚‰‰kD  rt        d‰› d‰› d�«      ‚t        j                  j                  d«      j	                  «       Šˆˆˆˆˆfd„}|�-|j                  «       j                  d«      j                  «       nt        |«      D �cg c]  }‰‘Œ c}}t        j                  |‰ft        ¬	«      }	g }
 |‰«      }|d
k(  r|	S |D ]¯  } ||«      }t        j                  j                  t        j                  |‰dz
  z
  «      |d¬«      }t        |«      d
k(  r‰dz
  }n|d
   }t        j                  |t        j                  ||z
  t        j                   ¬	«      |z  g«      }|
j#                  |«       Œ± t        j$                  |
«      }
t        j&                  |
dd…dd…df   ||‰f«      }
|
j)                  ||‰z  «      }
t        j                  ‰«      dddd…f   }t        j&                  |||‰f«      j)                  ||‰z  «      }|
|z   }
|
j+                  «       ‰dz
  kD  r‰dz
  |
|
‰dz
  kD  <   t        j,                  |	|
dd«       |	S c c}w )an  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`c                 óœ   •— t        ‰| z  ‰z  ‰z   «      }t        |‰«      }|‰z  ‰kD  r‰‰z  }| ‰dz
  z
  |k  rt        | ‰dz
  z
  d«      }|S )z;Given input length, compute how many spans should be maskedr   r   )rÜ   Úmax)r|  Únum_masked_spanÚepsilonr•  r”  r–  r\  s     €€€€€r/   Úcompute_num_masked_spanz6_compute_mask_indices.<locals>.compute_num_masked_spanC  so   ø€ ä˜i¨,Ñ6¸ÑDÀwÑNÓOˆÜ˜o¨yÓ9ˆð ˜[Ñ(¨?Ò:Ø-°Ñ<ˆOð ˜;¨™?Ñ+¨oÒ=Ü! ,°+À±/Ñ"BÀAÓFˆOàÐr.   Nrz   ©r‚  r   F)Úreplace)r‹   ÚnpÚrandomr  ÚitemÚdetachrM  Útolistr‰   r…  rÞ   Úchoicer†  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_torÒ   rš  Úput_along_axis)rÎ   r”  r•  r®   r–  r[  r�  rý   rv  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanr|  r›  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsrœ  r\  s    `` `            @@r/   Ú_compute_mask_indicesr´    s­  ü€ ð0 #(Ñ€J�à�Q‚ÜÐAÓBÐBà�_Ò$ÜØ]Ð^iÐ]jØ& Ð&7°qð:ó
ð 	
ô �i‰i�n‰n˜QÓ×$Ñ$Ó&€G÷ð ð$ Ð%ð 	×ÑÓ×#Ñ# BÓ'×.Ñ.Ô0ä',¨ZÓ'8Ö9 !ŠoÒ9ð ô —H‘H˜j¨/Ð:Ä$ÔG€MØÐá1°/ÓBÐà˜aÒØÐà%ò 5ˆá1°,Ó?ˆô ŸI™I×,Ñ,Ü�I‰I�l k°A¡oÑ6Ó7¸ÐRWð -ó 
Ðô Ð Ó! QÒ&ð -¨qÑ0‰Nà.¨qÑ1ˆNäŸN™NØ¤§¡Ð(;¸oÑ(MÔUW×U]ÑU]Ô ^ÐaoÑ oÐpó
Ðð 	×!Ñ!Ð"3Õ4ð/5ô2 Ÿ™Ð"4Ó5Ðô Ÿ™Øš1ša ˜:Ñ&¨Ð5HÈ+Ð(VóÐð ,×3Ñ3°JÐ@SÐVaÑ@aÓbÐô �i‰i˜Ó$ T¨4² ]Ñ3€GÜ�o‰o˜g¨
Ð4GÈÐ'UÓV×^Ñ^ØÐ'¨+Ñ5ó€Gð ,¨gÑ5Ðð ×ÑÓ /°AÑ"5Ò5ØGVÐYZÑGZÐÐ-°À!Ñ0CÑCÑDô ×Ñ�mÐ%7¸¸BÔ?àÐùòw :s   Â$	I+c                   ó&  ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  fd„Ze		 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  deez  fd„«       Zˆ xZS )ÚUniSpeechModelr^   c                 ó²  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |j                  dkD  s|j                  dkD  rEt        j                  t        j                  |j                  «      j                  «       «      | _        |j                   rt#        |«      | _        nt'        |«      | _        | j)                  «        y ©NrÍ   )r4   r5   r^   r„   Úfeature_extractorr�   Úfeature_projectionÚmask_time_probÚmask_feature_probrM   rF  r)   rß   rO   rn  Úmasked_spec_embedÚdo_stable_layer_normr;  Úencoderr   Ú	post_initr§   s     €r/   r5   zUniSpeechModel.__init__™  s¢   ø€ Ü‰Ñ˜Ô ØˆŒÜ!8¸Ó!@ˆÔÜ"<¸VÓ"DˆÔà× Ñ  3Ò&¨&×*BÑ*BÀSÒ*HÜ%'§\¡\´%·,±,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"à×&Ò&Ü:¸6ÓBˆD�Lä+¨FÓ3ˆDŒLð 	�‰Õr.   Nr#   Úmask_time_indicesr®   c                 óÎ  — t        | j                  dd«      s|S |j                  «       \  }}}|�)| j                  j	                  |j
                  «      ||<   nË| j                  j                  dkD  r²| j                  r¦t        ||f| j                  j                  | j                  j                  || j                  j                  ¬«      }t        j                  ||j                  t        j                  ¬«      }| j                  j	                  |j
                  «      ||<   | j                  j                  dkD  r¨| j                  rœt        ||f| j                  j                  | j                  j                   | j                  j"                  ¬«      }t        j                  ||j                  t        j                  ¬«      }|dd…df   j%                  d|d«      }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )r”  r•  r®   r–  )r  r‚  )r”  r•  r–  rz   )r7  r^   rµ   r½  r  r‚  r»  r™   r´  Úmask_time_lengthÚmask_time_min_masksr)   r'  r  rÞ   r¼  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)r7   r#   rÁ  r®   r[  r\  rO   Úmask_feature_indicess           r/   Ú_mask_hidden_statesz"UniSpeechModel._mask_hidden_statesª  sš  € ô �t—{‘{Ð$8¸$Ô?Ø Ð ð 4A×3EÑ3EÓ3GÑ0ˆ
�O [àÐ(à/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ò,Ø�[‰[×'Ñ'¨!Ò+°·²Ü 5Ø˜_Ð-ØŸ+™+×4Ñ4Ø ŸK™K×8Ñ8Ø-ØŸ+™+×9Ñ9ô!Ðô !&§¡Ð->À}×G[ÑG[Ôch×cmÑcmÔ nÐØ/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ñ,à�;‰;×(Ñ(¨1Ò,°·²ä#8Ø˜[Ð)ØŸ+™+×7Ñ7Ø ŸK™K×;Ñ;ØŸ+™+×<Ñ<ô	$Ð ô $)§<¡<Ð0DÈ]×MaÑMaÔin×isÑisÔ#tÐ Ø#7º¸4¸Ñ#@×#GÑ#GÈÈOÐ]_Ó#`Ð Ø23ˆMÐ.Ñ/àÐr.   rš   rÊ   r  r	  rË   c                 ó
  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      }|j                  dd«      }|�| j                  |j                  d   |«      }| j                  |«      \  }	}| j                  |	||¬«      }	| j                  |	||||¬«      }
|
d   }	|s
|	|f|
dd z   S t        |	||
j                  |
j                  ¬«      S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        Nr   r3   )rÁ  r®   ©r®   rÊ   r  r	  r   )r  Úextract_featuresr#   r$   )r^   rÊ   r  r	  r¹  r`   rŠ  rÎ   rº  rÊ  r¿  ÚUniSpeechBaseModelOutputr#   r$   )r7   rš   r®   rÁ  rÊ   r  r	  r°   rÍ  r#   Úencoder_outputss              r/   r=   zUniSpeechModel.forwardØ  s@  € ð  2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐàÐ%à!×DÑDÐEU×E[ÑE[Ð\]ÑE^Ð`nÓoˆNà*.×*AÑ*AÐBRÓ*SÑ'ˆÐ'Ø×0Ñ0ØÐ->È~ð 1ó 
ˆð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Ñ*ˆáØ!Ð#3Ð4°ÀqÀrÐ7JÑJÐJä'Ø+Ø-Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r.   r  ©NNNNN)r%   r&   r'   r   r5   r)   r*   r“  rÊ  r   rß   rÞ   r,   rÎ  r=   r?   r@   s   @r/   r¶  r¶  —  sí   ø„ ð˜õ ð( 7;Ø26ñ	,à×(Ñ(ð,ð !×,Ñ,¨tÑ3ð,ð ×(Ñ(¨4Ñ/ó	,ð\ ð /3Ø6:Ø)-Ø,0Ø#'ñ3
à—l‘l TÑ)ð3
ð Ÿ™ tÑ+ð3
ð !×,Ñ,¨tÑ3ð	3
ð
   $™;ð3
ð # T™kð3
ð ˜D‘[ð3
ð 
Ð)Ñ	)ò3
ó ô3
r.   r¶  zZ
    UniSpeech Model with a vector-quantization module and ctc loss for pre-training.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zdefd„Zd„ Ze	 dde	j                  de	j                  de	j                  defd	„«       Ze	 	 	 	 dde	j                  d
z  de	j                  d
z  ded
z  ded
z  ded
z  deez  fd„«       Zˆ xZS )ÚUniSpeechForPreTrainingr^   c                 ó.  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t        |«      | _	        t	        j                  |j                  |j                  «      | _        t	        j                  |j                  |j                  «      | _        t	        j                  |j                  |j                   «      | _        t	        j
                  |j$                  «      | _        | j)                  «        y rp   )r4   r5   r¶  rd  rM   r¤   Úfeat_quantizer_dropoutÚdropout_featuresr@  Ú	quantizerr¢   rE  Úproj_codevector_dimÚ	project_qrO   Úproject_hidÚnum_ctc_classesÚctc_projÚfinal_dropoutr¦   rÀ  r§   s     €r/   r5   z UniSpeechForPreTraining.__init__  s¼   ø€ Ü‰Ñ˜Ô Ü'¨Ó/ˆŒÜ "§
¡
¨6×+HÑ+HÓ IˆÔä7¸Ó?ˆŒÜŸ™ 6×#8Ñ#8¸&×:TÑ:TÓUˆŒÜŸ9™9 V×%?Ñ%?À×ASÑASÓTˆÔäŸ	™	 &×"4Ñ"4°f×6LÑ6LÓMˆŒÜ—z‘z &×"6Ñ"6Ó7ˆŒð 	�‰Õr.   rI  c                 ó&   — || j                   _        y)zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)rÖ  rI  )r7   rI  s     r/   Úset_gumbel_temperaturez.UniSpeechForPreTraining.set_gumbel_temperature$  s   € ð &1ˆ�‰Õ"r.   c                 óL   — | j                   j                  j                  «        y©z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N©rd  r¹  r—   ©r7   s    r/   Úfreeze_feature_encoderz.UniSpeechForPreTraining.freeze_feature_encoder*  ó   € ð
 	�‰×(Ñ(×;Ñ;Õ=r.   Útarget_featuresÚnegative_featuresÚpredicted_featuresc                 óÌ   — t        j                  | |gd¬«      } t        j                  |j                  «       | j                  «       d¬«      }|j	                  | «      }||z  }|S )zé
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r³   rz   )r)   ÚcatÚcosine_similarityrÝ   rW  )rå  ræ  rç  rI  Úlogitss        r/   Úcompute_contrastive_logitsz2UniSpeechForPreTraining.compute_contrastive_logits1  sa   € ô  Ÿ)™) _Ð6GÐ$HÈaÔPˆä×(Ñ(Ð);×)AÑ)AÓ)CÀ_×EZÑEZÓE\ÐbdÔeˆØ—‘ Ó0ˆð ˜+Ñ%ˆØˆr.   Nrš   r®   rÊ   r  r	  rË   c                 óþ  — |�|n| j                   j                  }| j                  |||||¬«      }|d   }| j                  |d   «      }	| j	                  |	«      \  }
}| j                  |
j                  | j
                  j                  j                  «      «      }
| j                  |
«      }
t        j                  |j                  d«      |j                  d«      «      j                  | j                   j                  «      }|j                  dd«      }t        j                   |«      j#                  «       j                  |j$                  «      }|j                  dd«      }|j'                  d«      }|j)                  |d«      |
j)                  | d«      z   }| j+                  |«      }| j-                  |«      }d}|s|�|||
|f|dd z   S ||
|f|dd z   S t/        |||
||j0                  |j2                  ¬«      S )	a›  
        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```NrÌ  r   r   rz   rÍ   r3   )r   r    r!   r"   r#   r$   )r^   r	  rd  rÕ  rÖ  rØ  r  rI   r‚  rÙ  r)   Úemptyrµ   Úfill_Úreplace_probr`   Ú	bernoullirÞ   r  r  Úmasked_fillr¦   rÛ  r   r#   r$   )r7   rš   r®   rÊ   r  r	  r°   rþ   Útransformer_featuresrÍ  Úquantized_featuresr"   Úprob_replace_matrixÚsampled_replace_matrixrë  r   s                   r/   r=   zUniSpeechForPreTraining.forwardE  s  € ð, &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—.‘.ØØ)Ø/Ø!5Ø#ð !ó 
ˆð  ' q™zÐð  ×0Ñ0°¸±Ó<ÐØ48·N±NÐCSÓ4TÑ1ÐÐ1ð "Ÿ^™^Ð,>×,AÑ,AÀ$Ç.Á.×BWÑBW×B]ÑB]Ó,^Ó_ÐØ!×-Ñ-Ð.@ÓAÐä#Ÿk™kÐ*>×*CÑ*CÀAÓ*FÐH\×HaÑHaÐbcÓHdÓe×kÑkØ�K‰K×$Ñ$ó
Ðð 2×;Ñ;¸A¸qÓAÐÜ!&§¡Ð1DÓ!E×!JÑ!JÓ!L×!OÑ!OÐPd×PkÑPkÓ!lÐØ!7×!AÑ!AÀ!ÀQÓ!GÐØ!7×!AÑ!AÀ"Ó!EÐØ%×1Ñ1Ð2HÈ#ÓNØ×*Ñ*Ð,BÐ+BÀCÓHñ
ˆð
 —‘˜fÓ%ˆØ—‘˜vÓ&ˆð ˆÙØÐØÐ2Ð4FÐH]Ð^ÐahÐijÐikÐalÑlÐlØ(Ð*<Ð>SÐTÐW^Ð_`Ð_aÐWbÑbÐbä,ØØ1Ø'9Ø"7Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   )r   )NNNN)r%   r&   r'   r   r5   rÜ   rÞ  rã  ra  r)   r*   rì  r   rß   rÞ   r,   r   r=   r?   r@   s   @r/   rÒ  rÒ    s÷   ø„ ð˜õ ð1°#ó 1ò>ð ð
 ñ	Ø×*Ñ*ðà ×,Ñ,ðð "×-Ñ-ðð ò	ó ðð& ð /3Ø)-Ø,0Ø#'ñE
à—l‘l TÑ)ðE
ð Ÿ™ tÑ+ðE
ð   $™;ð	E
ð
 # T™kðE
ð ˜D‘[ðE
ð 
Ð.Ñ	.òE
ó ôE
r.   rÒ  r3   zq
    UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    c                   óÞ   ‡ — e Zd Zddedz  fˆ fd„Zd„ Zd„ Zd„ Ze	 	 	 	 	 dde	j                  dz  de	j                  dz  d	edz  d
edz  dedz  de	j                  dz  deez  fd„«       Zˆ xZS )ÚUniSpeechForCTCNÚtarget_langc                 ó®  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        || _        |j                  €t        d| j                  › d�«      ‚t        |d«      r|j                  r|j                  n|j                  }t	        j                   ||j                  «      | _        | j%                  «        y)a3  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng' by
            default.
        NzYou are trying to instantiate zø with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `UniSpeechForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r4   r5   r¶  rd  rM   r¤   rÜ  r¦   rù  Ú
vocab_sizer‹   r9   rS   rû  Úoutput_hidden_sizerO   r¢   Úlm_headrÀ  )r7   r^   rù  rý  r9   s       €r/   r5   zUniSpeechForCTC.__init__—  s¿   ø€ ô 	‰Ñ˜Ô ä'¨Ó/ˆŒÜ—z‘z &×"6Ñ"6Ó7ˆŒà&ˆÔà×ÑÐ$ÜØ0°·±Ð0@ð AHð Hóð ô *1°¸Ô)GÈF×L^ÒL^ˆF×%Ò%Ðdj×dvÑdvð 	ô —y‘yÐ!3°V×5FÑ5FÓGˆŒð 	�‰Õr.   c                 ó8  — t        «       t        j                  d«      k(  ry| j                  }|�&t	        | j
                  dd«      €t        d|› d�«      ‚|€-t	        | j
                  dd«      �t        j                  d«       y|�| j                  |d¬«       yy)	a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        ÚmetaNr+  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   r)   r  rù  r7  r^   r‹   ÚloggerÚinfoÚload_adapter)r7   r°   rù  s      r/   Útie_weightszUniSpeechForCTC.tie_weights´  sœ   € ô 6Ó7¼5¿<¹<ÈÓ;OÒOØð ×&Ñ&ˆàÐ"¤w¨t¯{©{Ð<NÐPTÓ'UÐ']ÜÐ:¸;¸-ÐGtÐuÓvÐvØÐ ¤W¨T¯[©[Ð:LÈdÓ%SÐ%_Ü�K‰KÐCÕDØÐ$Ø×Ñ˜k°dÐÕ;ð %r.   c                 óL   — | j                   j                  j                  «        yrà  rá  râ  s    r/   rã  z&UniSpeechForCTC.freeze_feature_encoderÌ  rä  r.   c                 óP   — | j                   j                  «       D ]	  }d|_        Œ y©zÒ
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN©rd  r“   r”   r•   s     r/   Úfreeze_base_modelz!UniSpeechForCTC.freeze_base_modelÓ  ó(   € ð
 —^‘^×.Ñ.Ó0ò 	(ˆEØ"'ˆEÕñ	(r.   rš   r®   rÊ   r  r	  ÚlabelsrË   c           
      ó¤  — |�|n| j                   j                  }|�I|j                  «       | j                   j                  k\  r"t	        d| j                   j                  › �«      ‚| j                  |||||¬«      }|d   }	| j                  |	«      }	| j                  |	«      }
d}|��b|�|n$t        j                  |t        j                  ¬«      }| j                  |j                  d«      «      j                  t        j                  «      }|dk\  }|j                  d«      }|j                  |«      }t        j                   j#                  |
dt        j$                  ¬«      j'                  dd«      }t        j(                  j*                  j-                  d	¬
«      5  t        j                   j/                  ||||| j                   j0                  | j                   j2                  | j                   j4                  ¬«      }ddd«       |s|
f|t6        d z   }|�|f|z   S |S t9        ||
|j:                  |j<                  ¬«      S # 1 sw Y   ŒExY w)aà  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nz$Label values must be <= vocab_size: rÌ  r   rž  rz   )rK   r‚  r   F)Úenabled)ÚblankÚ	reductionÚzero_infinity©r   rë  r#   r$   )r^   r	  rš  rü  r‹   rd  r¦   rþ  r)   Ú	ones_liker„  r  rM  r  Úmasked_selectrM   r·   Úlog_softmaxÚfloat32r`   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityÚ_HIDDEN_STATES_START_POSITIONr   r#   r$   )r7   rš   r®   rÊ   r  r	  r  r°   rþ   r#   rë  r   rv  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probsÚoutputs                     r/   r=   zUniSpeechForCTC.forwardÛ  s'  € ð$ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ &§*¡*£,°$·+±+×2HÑ2HÒ"HÜÐCÀDÇKÁK×DZÑDZÐC[Ð\Ó]Ð]à—.‘.ØØ)Ø/Ø!5Ø#ð !ó 
ˆð   ™
ˆØŸ™ ]Ó3ˆà—‘˜mÓ,ˆàˆØÑð #1Ð"<‘Ä%Ç/Á/ÐR^Ôfk×fpÑfpÔBqð ð !×AÑAÀ.×BTÑBTÐUWÓBXÓY×\Ñ\Ô]b×]gÑ]gÓhˆMð ! A™+ˆKØ(Ÿ_™_¨RÓ0ˆNØ &× 4Ñ 4°[Ó AÐô Ÿ™×1Ñ1°&¸bÌÏÉÐ1ÓV×`Ñ`ÐabÐdeÓfˆIä—‘×%Ñ%×+Ñ+°EÐ+Ó:ñ 	Ü—}‘}×-Ñ-ØØ%Ø!Ø"ØŸ+™+×2Ñ2Ø"Ÿk™k×<Ñ<Ø"&§+¡+×"?Ñ"?ð .ó �÷	ñ Ø�Y Ô)FÐ)GÐ!HÑHˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØ˜f°G×4IÑ4IÐV]×VhÑVhô
ð 	
÷	ð 	ús   ÆA#IÉIrp   rÐ  )r%   r&   r'   ré   r5   r  rã  r
  r   r)   rß   rÞ   r,   r   r=   r?   r@   s   @r/   rø  rø  ‘  sÀ   ø„ ñ¨C°$©Jõ ò:<ò0>ò(ð ð /3Ø)-Ø,0Ø#'Ø&*ñE
à—l‘l TÑ)ðE
ð Ÿ™ tÑ+ðE
ð   $™;ð	E
ð
 # T™kðE
ð ˜D‘[ðE
ð —‘˜tÑ#ðE
ð 
�Ñ	òE
ó ôE
r.   rø  z˜
    UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   óÊ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  de	dz  d	e	dz  d
ej                  dz  de
ez  fd„«       Zˆ xZS )Ú"UniSpeechForSequenceClassificationc                 óü  •— t         ‰| �  |«       t        |d«      r|j                  rt	        d«      ‚t        |«      | _        |j                  dz   }|j                  r0t        j                  t        j                  |«      |z  «      | _        t        j                  |j                  |j                   «      | _        t        j                  |j                   |j$                  «      | _        | j)                  «        y )Nrû  z`Sequence classification does not support the use of UniSpeech adapters (config.add_adapter=True)r   )r4   r5   rS   rû  r‹   r¶  rd  r  Úuse_weighted_layer_sumrM   rF  r)   r¨  Úlayer_weightsr¢   rO   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierrÀ  )r7   r^   Ú
num_layersr9   s      €r/   r5   z+UniSpeechForSequenceClassification.__init__+  sÀ   ø€ Ü‰Ñ˜Ô ä�6˜=Ô)¨f×.@Ò.@ÜØróð ô (¨Ó/ˆŒØ×-Ñ-°Ñ1ˆ
Ø×(Ò(Ü!#§¡¬e¯j©j¸Ó.DÀzÑ.QÓ!RˆDÔÜŸ™ 6×#5Ñ#5°v×7RÑ7RÓSˆŒÜŸ)™) F×$?Ñ$?À×ARÑARÓSˆŒð 	�‰Õr.   c                 óL   — | j                   j                  j                  «        yrà  rá  râ  s    r/   rã  z9UniSpeechForSequenceClassification.freeze_feature_encoder<  rä  r.   c                 óP   — | j                   j                  «       D ]	  }d|_        Œ yr  r	  r•   s     r/   r
  z4UniSpeechForSequenceClassification.freeze_base_modelC  r  r.   Nrš   r®   rÊ   r  r	  r  rË   c                 ó<  — |�|n| j                   j                  }| j                   j                  rdn|}| j                  |||||¬«      }| j                   j                  rr|t           }	t        j                  |	d¬«      }	t        j                  j                  | j                  d¬«      }
|	|
j                  ddd«      z  j                  d¬«      }	n|d   }	| j                  |	«      }	|€|	j                  d¬«      }n‰| j                  |	j                   d   |«      }|j#                  d«      j%                  dd|	j                   d   «      }d	|	| <   |	j                  d¬«      |j                  d¬«      j                  dd«      z  }| j'                  |«      }d}|�Ft)        «       } ||j                  d| j                   j*                  «      |j                  d«      «      }|s|f|t        d z   }|�|f|z   S |S t-        |||j.                  |j0                  ¬
«      S )a  
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
            into a tensor of type `torch.FloatTensor`. See [`UniSpeechProcessor.__call__`] for details.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTrÌ  r   r³   rz   r   r3   rÍ   r  )r^   r	  r'  rd  r  r)   ÚstackrM   r·   r¸   r(  rÏ   rM  r*  rK  rŠ  rÎ   r  r  r,  r   r+  r   r#   r$   )r7   rš   r®   rÊ   r  r	  r  r°   rþ   r#   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskrë  r   Úloss_fctr#  s                     r/   r=   z*UniSpeechForSequenceClassification.forwardK  s  € ð0 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ'+§{¡{×'IÒ'I™tÐOcÐà—.‘.ØØ)Ø/Ø!5Ø#ð !ó 
ˆð �;‰;×-Ò-Ø#Ô$AÑBˆMÜ!ŸK™K¨¸1Ô=ˆMÜŸ=™=×0Ñ0°×1CÑ1CÈÐ0ÓLˆLØ*¨\×->Ñ->¸rÀ1ÀaÓ-HÑH×MÑMÐRSÐMÓT‰Mà# A™JˆMàŸ™ }Ó5ˆØÐ!Ø)×.Ñ.°1Ð.Ó5‰Mà×BÑBÀ=×CVÑCVÐWXÑCYÐ[iÓjˆLØ".×"8Ñ"8¸Ó"<×"CÑ"CÀAÀqÈ-×J]ÑJ]Ð^_ÑJ`Ó"aÐØ25ˆMÐ.Ð.Ñ/Ø)×-Ñ-°!Ð-Ó4°|×7GÑ7GÈAÐ7GÓ7N×7SÑ7SÐTVÐXYÓ7ZÑZˆMà—‘ Ó/ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDáØ�Y Ô)FÐ)GÐ!HÑHˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   rÐ  )r%   r&   r'   r5   rã  r
  r   r)   rß   rÞ   r,   r   r=   r?   r@   s   @r/   r%  r%  $  s±   ø„ ôò">ò(ð ð /3Ø)-Ø,0Ø#'Ø&*ñC
à—l‘l TÑ)ðC
ð Ÿ™ tÑ+ðC
ð   $™;ð	C
ð
 # T™kðC
ð ˜D‘[ðC
ð —‘˜tÑ#ðC
ð 
Ð)Ñ	)òC
ó ôC
r.   r%  )rø  rÒ  r%  r¶  rc  r¸  r;   )Pro  Úcollections.abcr   Údataclassesr   Únumpyr   r)   Útorch.nnrM   r   Ú r   rk  Úactivationsr   Úintegrations.deepspeedr	   Úintegrations.fsdpr
   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   rR   r   r   r   Úconfiguration_unispeechr   Ú
get_loggerr%   r  r   ÚModuler1   rB   rb   rs   r|   r„   r�   rß   rÝ   r¼   r¾   rá   rð   r   r)  r5  r;  r@  rc  r,   rÜ   r“  Úndarrayr´  rÎ  r¶  rÒ  r  rø  r%  Ú__all__r-   r.   r/   ú<module>rJ     s8  ðó* Ý $Ý !ã Û Ý Ý %å &Ý !Ý @Ý 7Ý 6Ý BÝ 9÷õ ÷ sÑ rÝ &ß @Ñ @Ý 4ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ô7 Kó 7ó óð7ô.˜BŸI™Iô ô* r§y¡yô *ôZÐ$>ô ô*Ð"<ô ô6Ð"<ô ô0&˜bŸi™iô &ôR1 §¡ô 1ð* !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8R/˜Ÿ™ô R/ôj˜2Ÿ9™9ô ô0!Ð6ô !ôHE
�r—y‘yô E
ôP §	¡	ô ô2+Ð+Eô +ô\I
 b§i¡iô I
ôXC' R§Y¡Yô C'ðL ôA˜ó Aó ðAðP /3ØñtØ��c�‰?ðtàðtð ðtð ×$Ñ$ tÑ+ð	tð
 ðtð ‡Z�Zótðn 3Ð ð ôt
Ð-ó t
ó ðt
ñn ðôô
w
Ð6ó w
óð
w
ðt !"Ð ñ ðôô
K
Ð.ó K
óð
K
ñ\ ðôôe
Ð)Aó e
óðe
òP�r.   