Ë
    ÛÓ^j¿,  ã                   ó°  — d Z ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZ ddlmZ dZ G d„ dej8                  «      Z G d„ de«      Z G d„ de«      Z G d„ dej8                  «      Z  G d„ de«      Z! G d„ de«      Z"e G d„ de«      «       Z# G d„ dee#«      Z$ G d„ de«      Z% G d „ d!e«      Z&g d"¢Z'y)#zPyTorch Hubert model.é    Né   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)ÚBaseModelOutput)ÚPreTrainedModel)Úauto_docstringé   )ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2ModelÚWav2Vec2SamePadLayeré   )ÚHubertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertPositionalConvEmbeddingc                 ó  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  dz  |j                  ¬«      | _        d | _        t        |dd«      r&t        j                  |j                  «      | _        �n¯t        j                  j                  }t        t        j                  j                  d«      r$t        j                  j                  j                  }t        «       �r(dd l}|j"                  j%                  | j                  j&                  d¬«      5   || j                  dd¬	«      | _        d d d «       t        | j                  d
«      rU| j                  j                  j&                  j(                  }| j                  j                  j&                  j*                  }n,| j                  j,                  }| j                  j.                  }|j"                  j1                  | |«       |j"                  j1                  | |«       n || j                  dd¬	«      | _        t3        |j
                  «      | _        t6        |j8                     | _        y # 1 sw Y   �Œ'xY w)Nr
   )Úkernel_sizeÚpaddingÚgroupsÚconv_pos_batch_normFÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr   Úhasattrr!   r   Ú	deepspeedÚzeroÚGatheredParametersr   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr   r/   r4   r5   Ú	__class__s         €út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/hubert/modular_hubert.pyr#   z&HubertPositionalConvEmbedding.__init__)   sÏ  ø€ Ü‰ÑÔÜ—I‘IØ×ÑØ×ÑØ×6Ñ6Ø×2Ñ2°aÑ7Ø×7Ñ7ô
ˆŒ	ð ˆŒÜ�6Ð0°%Ô8Ü Ÿn™n¨V×-?Ñ-?Ó@ˆDŽOäŸ(™(×.Ñ.ˆKÜ”r—x‘x×0Ñ0°-Ô@Ü Ÿh™h×7Ñ7×CÑC�ä)Õ+Û à—^‘^×6Ñ6°t·y±y×7GÑ7GÐWXÐ6ÓYñ MÙ +¨D¯I©I¸HÈ!Ô L�D”I÷Mä˜4Ÿ9™9Ð&8Ô9Ø#Ÿy™y×9Ñ9×@Ñ@×JÑJ�HØ#Ÿy™y×9Ñ9×@Ñ@×JÑJ‘Hà#Ÿy™y×1Ñ1�HØ#Ÿy™y×1Ñ1�HØ—‘×:Ñ:¸4ÀÔJØ—‘×:Ñ:¸4ÀÕJá'¨¯	©	¸ÀaÔH�”	ä)¨&×*HÑ*HÓIˆŒÜ  ×!?Ñ!?Ñ@ˆ�÷Mñ Mús   Å J Ê J
c                 óî   — |j                  dd«      }| j                  �| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|j                  dd«      }|S )Nr   r
   )Ú	transposer*   r)   r   r9   ©r:   Úhidden_statess     r=   Úforwardz%HubertPositionalConvEmbedding.forwardN   sn   € Ø%×/Ñ/°°1Ó5ˆØ�?‰?Ð&Ø ŸO™O¨MÓ:ˆMØŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆà%×/Ñ/°°1Ó5ˆØÐó    ©Ú__name__Ú
__module__Ú__qualname__r#   rB   Ú__classcell__©r<   s   @r=   r   r   (   s   ø„ ô#AöJ	rC   r   c                   ó   — e Zd Zy)r7   N©rE   rF   rG   © rC   r=   r7   r7   Z   ó   „ ØrC   r7   c                   ó   — e Zd Zy)ÚHubertFeatureEncoderNrK   rL   rC   r=   rO   rO   ^   rM   rC   rO   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚHubertFeatureProjectionc                 ón  •— t         ‰| �  «        |j                  | _        | j                  r3t        j                  |j
                  d   |j                  ¬«      | _        t        j                  |j
                  d   |j                  «      | _
        t        j                  |j                  «      | _        y )Néÿÿÿÿ)Úeps)r"   r#   Úfeat_proj_layer_normr$   Ú	LayerNormÚconv_dimÚlayer_norm_epsÚ
layer_normÚLinearr&   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©r:   r;   r<   s     €r=   r#   z HubertFeatureProjection.__init__c   s}   ø€ Ü‰ÑÔØ$*×$?Ñ$?ˆÔ!Ø×$Ò$Ü Ÿl™l¨6¯?©?¸2Ñ+>ÀF×DYÑDYÔZˆDŒOÜŸ)™) F§O¡O°BÑ$7¸×9KÑ9KÓLˆŒÜ—z‘z &×":Ñ":Ó;ˆ�rC   c                 ó„   — | j                   r| j                  |«      }| j                  |«      }| j                  |«      }|S )N)rU   rY   r[   r^   r@   s     r=   rB   zHubertFeatureProjection.forwardk   s;   € à×$Ò$Ø ŸO™O¨MÓ:ˆMØŸ™¨Ó6ˆØŸ™ ]Ó3ˆØÐrC   rD   rI   s   @r=   rQ   rQ   b   s   ø„ ô<örC   rQ   c                   ó   — e Zd Zy)ÚHubertEncoderNrK   rL   rC   r=   rb   rb   t   rM   rC   rb   c                   ó   — e Zd Zy)ÚHubertEncoderStableLayerNormNrK   rL   rC   r=   rd   rd   x   rM   rC   rd   c                   óÀ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZ ej                  «       ˆ fd„«       Zd	ej                   ez  fd
„Zdedej                   fd„Zˆ xZS )ÚHubertPreTrainedModelr;   ÚhubertÚinput_valuesÚaudioÚHubertEncoderLayerÚParametrizedConv1dTc                 óð  •— t         ‰| �  |«       t        |t        j                  «      �rt        «       rÈddl}t        |d«      rht        |d«      r\|j                  j                  |j                  |j                  gd¬«      5  t        j                  |j                  «       ddd«       no|j                  j                  |j                  d¬«      5  t        j                  |j                  «       ddd«       nt        j                  |j                  «       |j                  � t        j                   |j                  «       yyt        |t"        «      r-t        |d«      r t        j$                  |j&                  «       yyt        |t(        «      rHt        |d«      r;t        j*                  |j,                  d| j.                  j0                  d	z   z  «       yyy# 1 sw Y   ŒÌxY w# 1 sw Y   ŒØxY w)
zInitialize the weightsr   Nr5   r4   r   Úmasked_spec_embedÚlayer_weightsg      ð?r   )r"   Ú_init_weightsÚ
isinstancer$   r%   r   r/   r.   r0   r1   r5   r4   ÚinitÚkaiming_normal_r   ÚbiasÚzeros_ÚHubertModelÚuniform_rm   ÚHubertForSequenceClassificationÚ	constant_rn   r;   Únum_hidden_layers)r:   Úmoduler/   r<   s      €r=   ro   z#HubertPreTrainedModel._init_weightsˆ   s  ø€ ô 	‰Ñ˜fÔ%Ü�fœbŸi™iÕ(Ü)Ô+Û ä˜6 :Ô.´7¸6À:Ô3NØ"Ÿ™×:Ñ:¸F¿O¹OÈVÏ_É_Ð;]ÐmnÐ:Óoñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ð #Ÿ™×:Ñ:¸6¿=¹=ÐXYÐ:ÓZñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ô ×$Ñ$ V§]¡]Ô3à�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ä˜¤Ô,Ü�vÐ2Ô3Ü—‘˜f×6Ñ6Õ7ð 4ä˜Ô ?Ô@Ü�v˜Ô/Ü—‘˜v×3Ñ3°S¸D¿K¹K×<YÑ<YÐ\]Ñ<]Ñ5^Õ_ð 0ð A÷<ð <ú÷<ð <ús   Â G Ã G,Ç G)Ç,G5Úinput_lengthsc                 ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                 ó>   — t        j                  | |z
  |d¬«      dz   S )NÚfloor)Úrounding_moder   )ÚtorchÚdiv)Úinput_lengthr   Ústrides      r=   Ú_conv_out_lengthzPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length§   s"   € ô —9‘9˜\¨KÑ7¸ÈwÔWÐZ[Ñ[Ð[rC   )Úzipr;   Úconv_kernelÚconv_stride)r:   r{   r„   r   rƒ   s        r=   Ú _get_feat_extract_output_lengthsz6HubertPreTrainedModel._get_feat_extract_output_lengths¢   sQ   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð ÐrC   Úfeature_vector_lengthÚattention_maskc                 óä  — | j                  |j                  d«      «      j                  t        j                  «      }|j
                  d   }t        j                  ||f|j                  |j                  ¬«      }d|t        j                  |j
                  d   |j                  ¬«      |dz
  f<   |j                  dg«      j                  d«      j                  dg«      j                  «       }|S )NrS   r   )ÚdtypeÚdevicer   )r�   )rˆ   ÚsumÚtor€   ÚlongÚshapeÚzerosrŒ   r�   ÚarangeÚflipÚcumsumÚbool)r:   r‰   rŠ   Úoutput_lengthsÚ
batch_sizes        r=   Ú"_get_feature_vector_attention_maskz8HubertPreTrainedModel._get_feature_vector_attention_mask±   s×   € Ø×>Ñ>¸~×?QÑ?QÐRTÓ?UÓV×YÑYÔZ_×ZdÑZdÓeˆØ#×)Ñ)¨!Ñ,ˆ
äŸ™ØÐ.Ð/°~×7KÑ7KÐTb×TiÑTiô
ˆð uvˆœŸ™ ^×%9Ñ%9¸!Ñ%<À^×EZÑEZÔ[Ð]kÐnoÑ]oÐpÑqØ'×,Ñ,¨b¨TÓ2×9Ñ9¸"Ó=×BÑBÀBÀ4ÓH×MÑMÓOˆØÐrC   )rE   rF   rG   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr€   Úno_gradro   Ú
LongTensorÚintrˆ   r™   rH   rI   s   @r=   rf   rf   |   s‹   ø… àÓØ ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐà€U‡]�]ƒ_ó`ó ð`ð2¸e×>NÑ>NÐQTÑ>Tó ð
Èð 
Ð]b×]mÑ]m÷ 
rC   rf   c                   óÀ   ‡ — e Zd Zdefˆ fd„Zd„ Z	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  d	e	dz  d
e	dz  de
ez  fd„Zˆ xZS )ru   r;   c                 ó¶  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |j                  dkD  s|j                  dkD  rEt        j                  t        j                  |j                  «      j                  «       «      | _        |j                   rt#        |«      | _        nt'        |«      | _        | j)                  «        | `y )Ng        )r"   r#   r;   rO   Úfeature_extractorrQ   Úfeature_projectionÚmask_time_probÚmask_feature_probr$   Ú	Parameterr€   ÚTensorr&   rv   rm   Údo_stable_layer_normrd   Úencoderrb   Ú	post_initÚadapterr_   s     €r=   r#   zHubertModel.__init__¿   s§   ø€ Ü‰Ñ˜Ô ØˆŒÜ!5°fÓ!=ˆÔÜ"9¸&Ó"AˆÔà× Ñ  3Ò&¨&×*BÑ*BÀSÒ*HÜ%'§\¡\´%·,±,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"à×&Ò&Ü7¸Ó?ˆD�Lä(¨Ó0ˆDŒLð 	�‰Ôà‰LrC   c                 ó   — t        d«      ‚)NzNot needed for Hubert)ÚAttributeError)r:   s    r=   Úfreeze_feature_encoderz"HubertModel.freeze_feature_encoderÒ   s   € ÜÐ4Ó5Ð5rC   Nrh   rŠ   Úmask_time_indicesÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictÚreturnc                 óþ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      }|j                  dd«      }|�| j                  |j                  d   |«      }| j                  |«      }	| j                  |	|¬«      }	| j                  |	||||¬«      }
|
d   }	|s	|	f|
dd z   S t        |	|
j                  |
j                  ¬«      S )a1  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.

        Example:

        ```python
        >>> from transformers import AutoProcessor, HubertModel
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(example):
        ...     example["speech"] = example["audio"]["array"]
        ...     return example


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```Nr   r
   )rµ   )rŠ   r¶   r·   r¸   r   )Úlast_hidden_staterA   Ú
attentions)r;   r¶   r·   r¸   r¨   r?   r™   r‘   r©   Ú_mask_hidden_statesr¯   r   rA   r¼   )r:   rh   rŠ   rµ   r¶   r·   r¸   ÚkwargsÚextract_featuresrA   Úencoder_outputss              r=   rB   zHubertModel.forwardÕ   s,  € ðH 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐàÐ%à!×DÑDÐEU×E[ÑE[Ð\]ÑE^Ð`nÓoˆNà×/Ñ/Ð0@ÓAˆØ×0Ñ0°ÐRcÐ0ÓdˆàŸ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Ñ*ˆáØ!Ð# o°a°bÐ&9Ñ9Ð9äØ+Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
rC   )NNNNN)rE   rF   rG   r   r#   r´   r€   r­   ÚFloatTensorr–   Útupler   rB   rH   rI   s   @r=   ru   ru   ¾   s¤   ø„ ð˜|õ ò&6ð /3Ø6:Ø)-Ø,0Ø#'ñE
à—l‘l TÑ)ðE
ð Ÿ™ tÑ+ðE
ð !×,Ñ,¨tÑ3ð	E
ð
   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
�Ñ	 ÷E
rC   ru   c                   ó   — e Zd Zy)ÚHubertForCTCNrK   rL   rC   r=   rÄ   rÄ     rM   rC   rÄ   c                   ó   — e Zd Zy)rw   NrK   rL   rC   r=   rw   rw   !  rM   rC   rw   )rÄ   rw   ru   rf   )(Ú__doc__r€   Útorch.nnr$   Ú r   rq   Úactivationsr   Úintegrations.deepspeedr   Úmodeling_outputsr   Úmodeling_utilsr   r-   r	   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   Úconfiguration_hubertr   Ú_HIDDEN_STATES_START_POSITIONÚModuler   r7   rO   rQ   rb   rd   rf   ru   rÄ   rw   Ú__all__rL   rC   r=   ú<module>rÒ      sá   ðñ ã Ý å &Ý !Ý @Ý /Ý -Ý #÷÷ ñ õ /ð !"Ð ô/ B§I¡Iô /ôd	Ð-ô 	ô	Ð1ô 	ô˜bŸi™iô ô$	�Oô 	ô	Ð#Aô 	ð ô>˜Oó >ó ð>ôB\
�-Ð!6ô \
ô~	�>ô 	ô	Ð&Gô 	ò f�rC   