Ë
    ÜÓ^juY  ã                   óŽ  — d dl Z d dlZd dlmZ d dlmc mZ ddlmZ	 ddl
mZ ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZmZm Z  ddl!m"Z"  ejF                  e$«      Z% G d„ de«      Z& G d„ de«      Z' G d„ dejP                  «      Z) G d„ de«      Z* G d„ de«      Z+ G d„ de«      Z, G d„ dejP                  «      Z- G d„ dejP                  «      Z. G d„ dejP                  «      Z/ G d „ d!ee «      Z0eZ1 G d"„ d#e«      Z2 G d$„ d%e«      Z3 G d&„ d'e«      Z4 G d(„ d)e«      Z5 G d*„ d+e«      Z6g d,¢Z7y)-é    Né   )Úinitialization)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚWav2Vec2BaseModelOutput)ÚPreTrainedModel)Úloggingé   )	ÚWav2Vec2FeatureProjectionÚWav2Vec2FeedForwardÚ#Wav2Vec2ForAudioFrameClassificationÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2ForXVectorÚWav2Vec2ModelÚWav2Vec2PositionalConvEmbeddingÚWav2Vec2PreTrainedModelé   )ÚWavLMConfigc                   ó   — e Zd Zy)ÚWavLMPositionalConvEmbeddingN©Ú__name__Ú
__module__Ú__qualname__© ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/wavlm/modular_wavlm.pyr   r      ó   „ Ør   r   c                   ó   — e Zd Zy)ÚWavLMFeatureProjectionNr   r   r   r    r#   r#   #   r!   r   r#   c                   ó  ‡ — e Zd ZdZ	 	 	 	 ddededeez  dededefˆ fd„Z	 	 	 	 dd
ej                  dej                  d	z  dej                  d	z  dede
ej                  ej                  d	z  e
ej                     d	z  f   f
d„Zd
ej                  dej                  ej                  z  dej                  dede
ej                  ej                  f   f
d„Zdededej                  fd„Zdej                  dej                  fd„Zˆ xZS )ÚWavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚ	embed_dimÚ	num_headsÚdropoutÚnum_bucketsÚmax_distanceÚhas_relative_position_biasc                 ó  •— t         ‰| �  «        || _        || _        || _        ||z  | _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        t        j                  ||«      | _
        t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        || _        || _        t        j                   t#        j$                  d| j                  dd«      «      | _        t        j                  | j
                  d«      | _        |r0t        j*                  | j                  | j                  «      | _        y y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿r   é   )ÚsuperÚ__init__r&   r'   r(   Úhead_dimÚ
ValueErrorÚscalingÚnnÚLinearÚk_projÚv_projÚq_projÚout_projr)   r*   Ú	ParameterÚtorchÚonesÚgru_rel_pos_constÚgru_rel_pos_linearÚ	EmbeddingÚrel_attn_embed)Úselfr&   r'   r(   r)   r*   r+   Ú	__class__s          €r    r/   zWavLMAttention.__init__*   s7  ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒä—i‘i 	¨9Ó5ˆŒÜ—i‘i 	¨9Ó5ˆŒÜ—i‘i 	¨9Ó5ˆŒÜŸ	™	 )¨YÓ7ˆŒà&ˆÔØ(ˆÔä!#§¡¬e¯j©j¸¸D¿N¹NÈAÈqÓ.QÓ!RˆÔÜ"$§)¡)¨D¯M©M¸1Ó"=ˆÔá%Ü"$§,¡,¨t×/?Ñ/?ÀÇÁÓ"PˆDÕð &r   NÚhidden_statesÚattention_maskÚposition_biasÚoutput_attentionsÚreturnc                 ó   — |j                  «       \  }}}|€S| j                  ||«      }|j                  d«      j                  |ddd«      j	                  || j
                  z  ||«      }|j	                  |j                  dd | j
                  dfz   «      }	|	j                  dddd«      }	| j                  |	«      }
|
j	                  |	j                  dd dz   «      j                  d«      }
t        j                  |
«      j                  dd¬«      \  }}||| j                  z  d	z
  z  d
z   }|j	                  || j
                  z  dd«      |z  }|j	                  d||f«      }| j                  ||||«      \  }}|||fS )z'Attention layer with relative attentionNr   r   éÿÿÿÿr   r   )r   é   ©Údimç      ð?g       @)ÚsizeÚcompute_biasÚ	unsqueezeÚrepeatÚviewr'   ÚshapeÚpermuter=   Úsumr:   ÚsigmoidÚchunkr<   Útorch_multi_head_self_attention)r@   rB   rC   rD   rE   ÚindexÚbszÚtgt_lenÚ_Úgated_hidden_statesÚrelative_position_projÚgate_aÚgate_bÚgate_outputÚgated_position_biasÚattn_outputÚattn_weightss                    r    ÚforwardzWavLMAttention.forwardN   s­  € ð (×,Ñ,Ó.‰ˆˆW�að Ð Ø ×-Ñ-¨g°wÓ?ˆMà×'Ñ'¨Ó*×1Ñ1°#°q¸!¸QÓ?×DÑDÀSÈ4Ï>É>ÑEYÐ[bÐdkÓlð ð ,×0Ñ0°×1DÑ1DÀSÀbÐ1IÈTÏ^É^Ð]_ÐL`Ñ1`ÓaÐØ1×9Ñ9¸!¸QÀÀ1ÓEÐð "&×!8Ñ!8Ð9LÓ!MÐØ!7×!<Ñ!<Ð=P×=VÑ=VÐWZÐXZÐ=[Ð^dÑ=dÓ!e×!iÑ!iÐjlÓ!mÐô Ÿ™Ð'=Ó>×DÑDÀQÈBÐDÓO‰ˆ�Ø ¨×)?Ñ)?Ñ ?À#Ñ EÑFÈÑLˆð *×.Ñ.¨s°T·^±^Ñ/CÀRÈÓKÈmÑ[ÐØ1×6Ñ6¸¸GÀWÐ7MÓNÐà$(×$HÑ$HØ˜>Ð+>Ð@Qó%
Ñ!ˆ�\ð ˜L¨-Ð7Ð7r   ra   c                 óX  — |j                  dd«      x}x}}|�|j                  d«      nd}dx}	}
d}t        j                  |||| j                  | j
                  t        j                  dg«      t        j                  | j                  j                  | j                  j                  | j                  j                  f«      |	|
|| j                  | j                  j                  | j                  j                  | j                   |||d| j                  j                  | j                  j                  | j                  j                  ¬«      \  }}|j                  dd«      }|�C|dd…df   j#                  |j$                  dd | j
                  fz   |j$                  dd z   «      }||fS )zCsimple wrapper around torch's multi_head_attention_forward functionr   r   NFT)Úuse_separate_proj_weightÚq_proj_weightÚk_proj_weightÚv_proj_weight)Ú	transposeÚneÚFÚmulti_head_attention_forwardr&   r'   r:   ÚemptyÚcatr7   Úbiasr5   r6   r(   r8   ÚweightÚtrainingÚbroadcast_torR   )r@   rB   rC   ra   rE   ÚqueryÚkeyÚvalueÚkey_padding_maskÚbias_kÚbias_vÚadd_zero_attnrb   rc   s                 r    rW   z.WavLMAttention.torch_multi_head_self_attentionw   s�  € ð ,×5Ñ5°a¸Ó;Ð;ˆÐ;��eØ3AÐ3M˜>×,Ñ,¨QÔ/ÐSWÐð Ðˆ�Øˆô %&×$BÑ$BØØØØ�N‰NØ�N‰NÜ�K‰K˜˜ÓÜ�I‰I�t—{‘{×'Ñ'¨¯©×)9Ñ)9¸4¿;¹;×;KÑ;KÐLÓMØØØØ�L‰LØ�M‰M× Ñ Ø�M‰M×ÑØ�M‰MØØØØ%)ØŸ+™+×,Ñ,ØŸ+™+×,Ñ,ØŸ+™+×,Ñ,ô+%
Ñ!ˆ�\ð2 "×+Ñ+¨A¨qÓ1ˆàÐ#ð (ª¨4¨Ñ0×=Ñ=Ø×"Ñ" 2 AÐ&¨$¯.©.Ð):Ñ:¸\×=OÑ=OÐPQÐPRÐ=SÑSóˆLð ˜LÐ(Ð(r   Úquery_lengthÚ
key_lengthc                 óˆ  — t        j                  |t         j                  ¬«      d d …d f   }t        j                  |t         j                  ¬«      d d d …f   }||z
  }| j                  |«      }|j	                  | j
                  j                  j                  «      }| j                  |«      }|j                  g d¢«      }|S )N)Údtype)r   r   r   )	r:   ÚarangeÚlongÚ_relative_positions_bucketÚtor?   rq   ÚdevicerS   )r@   r{   r|   Úcontext_positionÚmemory_positionÚrelative_positionÚrelative_position_bucketÚvaluess           r    rN   zWavLMAttention.compute_bias®   s¢   € Ü Ÿ<™<¨¼E¿J¹JÔGÊÈ4ÈÑPÐÜŸ,™, z¼¿¹ÔDÀTÊ1ÀWÑMˆØ+Ð.>Ñ>ÐØ#'×#BÑ#BÐCTÓ#UÐ Ø#;×#>Ñ#>¸t×?RÑ?R×?YÑ?Y×?`Ñ?`Ó#aÐ Ø×$Ñ$Ð%=Ó>ˆØ—‘¢	Ó*ˆØˆr   Úrelative_positionsc                 ó$  — | j                   dz  }|dkD  j                  t        j                  «      |z  }t        j                  |«      }|dz  }||k  }t        j
                  |j                  «       |z  «      }|t        j
                  | j                  |z  «      z  }|||z
  z  }||z   j                  t        j                  «      }t        j                  |t        j                  ||dz
  «      «      }|t        j                  |||«      z  }|S )Nr   r   r   )r)   r‚   r:   r€   ÚabsÚlogÚfloatÚmathr*   ÚminÚ	full_likeÚwhere)r@   r‰   r)   Úrelative_bucketsÚ	max_exactÚis_smallÚrelative_positions_if_largeÚrelative_position_if_larges           r    r�   z)WavLMAttention._relative_positions_bucket¸   s  € Ø×&Ñ&¨!Ñ+ˆà.°Ñ2×6Ñ6´u·z±zÓBÀ[ÑPÐÜ"ŸY™YÐ'9Ó:Ðà 1Ñ$ˆ	Ø%¨	Ñ1ˆä&+§i¡iÐ0B×0HÑ0HÓ0JÈYÑ0VÓ&WÐ#Ø&AÄDÇHÁHÈT×M^ÑM^ÐajÑMjÓDkÑ&kÐ#Ø&AÀ[ÐS\ÑE\Ñ&]Ð#Ø&/Ð2MÑ&M×%QÑ%QÔRW×R\ÑR\Ó%]Ð"Ü%*§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2DÐF`ÓaÑaÐØÐr   )ç        i@  i   T©NNFr   )r   r   r   Ú__doc__Úintr�   Úboolr/   r:   ÚTensorÚtuplerd   ÚFloatTensorÚ
LongTensorÚ
BoolTensorrW   rN   r�   Ú__classcell__©rA   s   @r    r%   r%   '   s‰  ø„ ÙGð  #ØØØ+/ñ"Qàð"Qð ð"Qð ˜‘ð	"Qð
 ð"Qð ð"Qð %)õ"QðN /3Ø-1Ø"'Øñ'8à—|‘|ð'8ð Ÿ™ tÑ+ð'8ð —|‘| dÑ*ð	'8ð
  ð'8ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	Mó'8ðR5)à×(Ñ(ð5)ð ×(Ñ(¨5×+;Ñ+;Ñ;ð5)ð #×.Ñ.ð	5)ð
  ð5)ð 
ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4ó5)ðn¨ð ¸#ð À%×BSÑBSó ð ¸U×=NÑ=Nð  ÐSX×SdÑSd÷  r   r%   c                   ó   — e Zd Zy)ÚWavLMFeedForwardNr   r   r   r    r¤   r¤   Í   r!   r   r¤   c                   ó2   ‡ — e Zd Zddedefˆ fd„Zdd„Zˆ xZS )ÚWavLMEncoderLayerÚconfigr+   c                 óÚ  •— t         ‰| �  «        t        |j                  |j                  |j
                  |j                  |j                  |¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t!        |«      | _        t        j                  |j                  |j                  ¬«      | _        y ©N)r&   r'   r(   r)   r*   r+   ©Úeps©r.   r/   r%   Úhidden_sizeÚnum_attention_headsÚattention_dropoutr)   Úmax_bucket_distanceÚ	attentionr3   ÚDropoutÚhidden_dropoutr(   Ú	LayerNormÚlayer_norm_epsÚ
layer_normr¤   Úfeed_forwardÚfinal_layer_norm©r@   r§   r+   rA   s      €r    r/   zWavLMEncoderLayer.__init__Ò   ó¬   ø€ Ü‰ÑÔÜ'Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,Ø×*Ñ*Ø×3Ñ3Ø'Aô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ,¨VÓ4ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÕr   c                 óî   — |}| j                  |||||¬«      \  }}}| j                  |«      }||z   }| j                  |«      }|| j                  |«      z   }| j	                  |«      }||f}|r||fz  }|S )N©rC   rD   rE   rX   )r±   r(   r¶   r·   r¸   )	r@   rB   rC   rD   rE   rX   Úattn_residualrc   Úoutputss	            r    rd   zWavLMEncoderLayer.forwardá   sœ   € Ø%ˆØ59·^±^ØØ)Ø'Ø/Øð 6Dó 6
Ñ2ˆ�| ]ð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆàŸ™¨Ó6ˆà%¨×(9Ñ(9¸-Ó(HÑHˆØ×-Ñ-¨mÓ<ˆà  -Ð0ˆáØ˜�Ñ&ˆGàˆr   ©Tr˜   ©r   r   r   r   r›   r/   rd   r¡   r¢   s   @r    r¦   r¦   Ñ   s   ø„ ñ\˜{ð \Èõ \÷r   r¦   c                   ó2   ‡ — e Zd Zddedefˆ fd„Zdd„Zˆ xZS )Ú WavLMEncoderLayerStableLayerNormr§   r+   c                 óÚ  •— t         ‰| �  «        t        |j                  |j                  |j
                  |j                  |j                  |¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t!        |«      | _        t        j                  |j                  |j                  ¬«      | _        y r©   r¬   r¹   s      €r    r/   z)WavLMEncoderLayerStableLayerNorm.__init__û   rº   r   c                 óè   — |}| j                  |«      }| j                  ||||¬«      \  }}}| j                  |«      }||z   }|| j                  | j	                  |«      «      z   }||f}|r||fz  }|S )N)rC   rD   rE   )r¶   r±   r(   r·   r¸   )r@   rB   rC   rD   rE   r½   rc   r¾   s           r    rd   z(WavLMEncoderLayerStableLayerNorm.forward
  s•   € Ø%ˆØŸ™¨Ó6ˆØ59·^±^ØØ)Ø'Ø/ð	 6Dó 6
Ñ2ˆ�| ]ð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆØ%¨×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Ñ_ˆà  -Ð0ˆáØ˜�Ñ&ˆGàˆr   r¿   )NNFrÀ   r¢   s   @r    rÂ   rÂ   ú   s   ø„ ñ\˜{ð \Èõ \÷r   rÂ   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚWavLMEncoderc           
      ó¢  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        ||dk(  ¬«      ‘Œ c}«      | _        d| _        y c c}w ©Nrª   r   )r+   F)r.   r/   r§   r   Úpos_conv_embedr3   r´   r­   rµ   r¶   r²   r³   r(   Ú
ModuleListÚrangeÚnum_hidden_layersr¦   ÚlayersÚgradient_checkpointing©r@   r§   ÚirA   s      €r    r/   zWavLMEncoder.__init__   s�   ø€ Ü‰ÑÔØˆŒÜ:¸6ÓBˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÜUZÐ[a×[sÑ[sÓUtÖuÐPQÔ˜vÀ1ÈÁ6ÖKÒuó
ˆŒð ',ˆÕ#ùò vó   Â!Cc                 ó¦  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   | j                  |«      }	||	z   }| j	                  |«      }| j                  |«      }t        «       xs t        | «      }
d }t        | j                  «      D ]y  \  }}|r||fz   }t        j                  g «      }| j                  xr  |dkD  xr || j                  j                  k  }|r|
r ||||||¬«      }|d d \  }}|rd}|sŒq|d   fz   }Œ{ |r||fz   }|st        d„ |||fD «       «      S t!        |||¬	«      S )
Nr   rH   r   r   r   r¼   ©NNNc              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­w©Nr   ©Ú.0Úvs     r    ú	<genexpr>z'WavLMEncoder.forward.<locals>.<genexpr>a  ó   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š©Úlast_hidden_staterB   Ú
attentions)rO   rP   rR   rÉ   r¶   r(   r   r   Ú	enumeraterÍ   r:   Úrandrr   r§   Ú	layerdropr�   r   ©r@   rB   rC   rE   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚposition_embeddingsÚsynced_gpusrD   rÐ   ÚlayerÚdropout_probabilityÚskip_the_layerÚlayer_outputss                    r    rd   zWavLMEncoder.forward+  s¦  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6RˆØˆä! $§+¡+Ó.ò 	P‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]Òf¨q°1©uÒfÐ:MÐPT×P[ÑP[×PeÑPeÑ:eˆNÙ!¡[á %Ø!Ø#1Ø"/Ø&7Øô!�ð 0=¸R¸aÐ/@Ñ,�˜}áØ 2�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð1	Pñ4  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r   ©NFFT©r   r   r   r/   rd   r¡   r¢   s   @r    rÆ   rÆ     s   ø„ ô	,ð ØØ"Ø÷;
r   rÆ   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚWavLMEncoderStableLayerNormc           
      ó¢  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        ||dk(  ¬«      ‘Œ c}«      | _        d| _        y c c}w rÈ   )r.   r/   r§   r   rÉ   r3   r´   r­   rµ   r¶   r²   r³   r(   rÊ   rË   rÌ   rÂ   rÍ   rÎ   rÏ   s      €r    r/   z$WavLMEncoderStableLayerNorm.__init__j  s¦   ø€ Ü‰ÑÔØˆŒÜ:¸6ÓBˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mô ˜v×7Ñ7Ó8öàô 1°ÐUVÐZ[ÑU[Ö]òó
ˆŒð ',ˆÕ#ùòrÑ   c                 ó¤  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   | j                  |«      }	||	z   }| j	                  |«      }t        «       xs t        | «      }
d }t        | j                  «      D ]x  \  }}|r||fz   }t        j                  g «      }| j                  xr  |dkD  xr || j                  j                  k  }|r|
r |||||¬«      }|d d \  }}|rd}|sŒp|d   fz   }Œz | j                  |«      }|r||fz   }|st        d„ |||fD «       «      S t!        |||¬	«      S )
Nr   rH   r   r   r   )rC   rE   rD   rÓ   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÕ   r   rÖ   s     r    rÙ   z6WavLMEncoderStableLayerNorm.forward.<locals>.<genexpr>®  rÚ   rÛ   rÜ   )rO   rP   rR   rÉ   r(   r   r   rß   rÍ   r:   rà   rr   r§   rá   r¶   r�   r   râ   s                    r    rd   z#WavLMEncoderStableLayerNorm.forwardx  s¥  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6RˆØˆä! $§+¡+Ó.ò 	P‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]Òf¨q°1©uÒfÐ:MÐPT×P[ÑP[×PeÑPeÑ:eˆNÙ!¡[ñ !&Ø!Ø#1Ø&7Ø"/ô	!�ð 0=¸R¸aÐ/@Ñ,�˜}áØ 2�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð/	Pð2 Ÿ™¨Ó6ˆáØ 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ð;LÐYlô
ð 	
r   rî   rï   r¢   s   @r    rñ   rñ   i  s   ø„ ô,ð" ØØ"Ø÷9
r   rñ   c                   ó8   ‡ — e Zd ZdZˆ fd„Zed„ «       Zd„ Zˆ xZS )ÚWavLMGumbelVectorQuantizerz°
    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information.
    c                 ó0  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | j                  z  dk7  r&t        d|j                  › d| j                  › d�«      ‚t        j                  t        j                  d| j                  | j
                  z  |j                  | j                  z  «      «      | _        t        j                  |j                  d   | j                  | j
                  z  «      | _        d| _        y )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.r   rH   r   )r.   r/   Únum_codevector_groupsÚ
num_groupsÚnum_codevectors_per_groupÚnum_varsÚcodevector_dimr1   r3   r9   r:   rž   Úcodevectorsr4   Úconv_dimÚweight_projÚtemperature)r@   r§   rA   s     €r    r/   z#WavLMGumbelVectorQuantizer.__init__º  sî   ø€ Ü‰ÑÔØ ×6Ñ6ˆŒØ×8Ñ8ˆŒà× Ñ  4§?¡?Ñ2°aÒ7ÜØ)¨&×*?Ñ*?Ð)@ð A6Ø6:·o±oÐ5Fð G%ð%óð ô Ÿ<™<Ü×Ñ˜a §¡°4·=±=Ñ!@À&×BWÑBWÐ[_×[jÑ[jÑBjÓkó
ˆÔô Ÿ9™9 V§_¡_°RÑ%8¸$¿/¹/ÈDÏMÉMÑ:YÓZˆÔð ˆÕr   c                 óÄ   — | j                  d¬«      }t        j                  t        j                  t        j                  ||«      d¬«       «      j                  «       }|S )Nr   rJ   rH   )Úmeanr:   ÚexprT   Úxlogy)ÚprobsÚmarginal_probsÚ
perplexitys      r    Ú_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexityÏ  sI   € àŸ™¨˜Ó*ˆÜ—Y‘Y¤§	¡	¬%¯+©+°nÀnÓ*UÐ[]Ô ^Ð^Ó_×cÑcÓeˆ
ØÐr   c                 óâ  — |j                   \  }}}| j                  |«      }|j                  ||z  | j                  z  d«      }| j                  r t
        j                  j                  |j                  «       | j                  d¬«      }|j                  |«      }t        j                  |j                  ||z  | j                  d«      j                  «       d¬«      }| j                  |«      }n}|j                  d¬«      } |j                  |j                   Ž j!                  d|j                  dd«      d«      }|j                  ||z  | j                  d«      }| j                  |«      }|j                  ||z  d«      }|j#                  d«      | j$                  z  }	|	j                  ||z  | j                  | j&                  d«      }
|
j)                  d«      j                  ||d«      }
|
|fS )NrH   T)ÚtauÚhardrJ   r   rL   éþÿÿÿ)rR   rÿ   rQ   rù   rr   r3   Ú
functionalÚgumbel_softmaxr�   r   Útype_asr:   Úsoftmaxr  ÚargmaxÚ	new_zerosÚscatter_rO   rý   rû   rT   )r@   rB   Ú
batch_sizeÚsequence_lengthr­   Úcodevector_probsÚcodevector_soft_distr  Úcodevector_idxÚcodevectors_per_grouprý   s              r    rd   z"WavLMGumbelVectorQuantizer.forwardÕ  sÝ  € Ø3@×3FÑ3FÑ0ˆ
�O [ð ×(Ñ(¨Ó7ˆØ%×*Ñ*¨:¸Ñ+GÈ$Ï/É/Ñ+YÐ[]Ó^ˆà�=Š=ä!Ÿ}™}×;Ñ;¸M×<OÑ<OÓ<QÐW[×WgÑWgÐnrÐ;ÓsÐØ/×7Ñ7¸ÓFÐô $)§=¡=Ø×"Ñ" :°Ñ#?ÀÇÁÐRTÓU×[Ñ[Ó]Ðceô$Ð ð ×1Ñ1Ð2FÓG‰Jð +×1Ñ1°bÐ1Ó9ˆNØ6˜}×6Ñ6¸×8KÑ8KÐL×UÑUØ�N×'Ñ'¨¨AÓ.°ó Ðð  0×4Ñ4°ZÀ/Ñ5QÐSW×SbÑSbÐdfÓgÐà×1Ñ1Ð2BÓCˆJà+×0Ñ0°¸oÑ1MÈrÓRÐà 0× :Ñ :¸2Ó >À×AQÑAQÑ QÐØ+×0Ñ0°¸oÑ1MÈtÏÉÐ`d×`mÑ`mÐoqÓrˆØ!—o‘o bÓ)×.Ñ.¨z¸?ÈBÓOˆà˜JÐ&Ð&r   )	r   r   r   r™   r/   Ústaticmethodr  rd   r¡   r¢   s   @r    rö   rö   ´  s&   ø„ ñô
ð* ñó ðö
"'r   rö   c                   ót   — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej                  «       d„ «       Zd„ Zd	„ Zd
„ Zy)ÚWavLMPreTrainedModelr§   ÚwavlmÚinput_valuesÚaudioTFc           
      ó  — t        j                  | |«       t        |t        «      rut	        j
                  |j                  j                  dd¬«       t	        j                  |j                  j                  «       t	        j                  |j                  «       yt        |t        «      ržt	        j
                  |j                  j                  ddt        j                  d|j                  j                   d   |j                  j"                  z  z  «      z  ¬«       t	        j$                  |j                  j                  d«       yt        |t&        «      r‡t        j                  d|j(                  j*                  z  «      }t	        j                  |j(                  j                  | |¬«       t	        j                  |j(                  j                  | |¬«       yt        |t,        j.                  «      rŒt	        j0                  |j                  «       |j                  �`t        j                  |j2                  |j"                  |j                   d   z  z  «      }t	        j                  |j                  | |¬«       yyy)zInitialize the weightsr—   r   )r  Ústdr   r   )ÚaÚbN)r
   Ú_init_weightsÚ
isinstancerö   ÚinitÚnormal_rÿ   rq   Úzeros_rp   Úuniform_rý   r   ÚconvrŽ   ÚsqrtÚkernel_sizeÚin_channelsÚ	constant_r#   Ú
projectionÚin_featuresr3   ÚConv1dÚkaiming_normal_Úgroups)r@   ÚmoduleÚks      r    r$  z"WavLMPreTrainedModel._init_weights  s¸  € ô 	×%Ñ% d¨FÔ3ä�fÔ8Ô9Ü�L‰L˜×+Ñ+×2Ñ2¸À!ÕDÜ�K‰K˜×*Ñ*×/Ñ/Ô0Ü�M‰M˜&×,Ñ,Õ-Ü˜Ô <Ô=Ü�L‰LØ—‘×"Ñ"ØØœŸ	™	 ! v§{¡{×'>Ñ'>¸qÑ'AÀFÇKÁK×D[ÑD[Ñ'[Ñ"\Ó]Ñ]õô
 �N‰N˜6Ÿ;™;×+Ñ+¨QÕ/Ü˜Ô 6Ô7Ü—	‘	˜!˜f×/Ñ/×;Ñ;Ñ;Ó<ˆAÜ�M‰M˜&×+Ñ+×2Ñ2°q°b¸AÕ>Ü�M‰M˜&×+Ñ+×0Ñ0°Q°B¸!Ö<Ü˜¤§	¡	Ô*Ü× Ñ  §¡Ô/à�{‰{Ð&Ü—I‘I˜fŸm™m¨v×/AÑ/AÀF×DVÑDVÐWXÑDYÑ/YÑZÓ[�Ü—‘˜fŸk™k¨a¨R°1Ö5ð 'ð +r   c                 ó   — t        d«      ‚©NzNot needed for WavLM©ÚAttributeError©r@   s    r    Ú_get_adaptersz"WavLMPreTrainedModel._get_adapters  ó   € ÜÐ3Ó4Ð4r   c                 ó   — t        d«      ‚r7  r8  r:  s    r    Úinit_adapter_layersz(WavLMPreTrainedModel.init_adapter_layers"  r<  r   c                 ó   — t        d«      ‚r7  r8  r:  s    r    Úload_adapterz!WavLMPreTrainedModel.load_adapter%  r<  r   N)r   r   r   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr:   Úno_gradr$  r;  r>  r@  r   r   r    r  r  ú  sY   … ØÓØÐØ$€OØÐØ&*Ð#Ø ÐØ€NØÐà€U‡]�]ƒ_ñ6ó ð6ò45ò5ó5r   r  c                   ó   — e Zd Zy)Ú
WavLMModelNr   r   r   r    rK  rK  ,  r!   r   rK  c                   ó   — e Zd Zy)ÚWavLMForCTCNr   r   r   r    rM  rM  0  r!   r   rM  c                   ó   — e Zd Zy)ÚWavLMForSequenceClassificationNr   r   r   r    rO  rO  4  r!   r   rO  c                   ó   — e Zd Zy)Ú WavLMForAudioFrameClassificationNr   r   r   r    rQ  rQ  8  r!   r   rQ  c                   ó   — e Zd Zy)ÚWavLMForXVectorNr   r   r   r    rS  rS  <  r!   r   rS  )rQ  rM  rO  rS  rK  r  )8rŽ   r:   Útorch.nnr3   Útorch.nn.functionalr  rl   Ú r   r&  Úintegrations.deepspeedr   Úintegrations.fsdpr   Úmodeling_layersr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   Úutilsr   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   Úconfiguration_wavlmr   Ú
get_loggerr   Úloggerr   r#   ÚModuler%   r¤   r¦   rÂ   rÆ   rñ   rö   r  ÚWavLMBaseModelOutputrK  rM  rO  rQ  rS  Ú__all__r   r   r    ú<module>rd     s>  ðÛ ã Ý ß Ð å &Ý @Ý 7Ý 9ß HÝ -Ý ÷
÷ 
õ 
õ -ð 
ˆ×	Ñ	˜HÓ	%€ô	Ð#Bô 	ô	Ð6ô 	ôc �R—Y‘Yô c ôL	Ð*ô 	ô&Ð2ô &ôR"Ð'Aô "ôJG
�2—9‘9ô G
ôTH
 "§)¡)ô H
ôVC' §¡ô C'ôL,5˜?Ð,Cô ,5ð^ /Ð ô	�ô 	ô	�.ô 	ô	Ð%Fô 	ô	Ð'Jô 	ô	Ð(ô 	ò�r   