Ë
    ÜÓ^jÚE  ã                   ój  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z" dZ# G d„ de«      Z$ G d„ de«      Z% G d„ de«      Z& G d„ dejN                  «      Z( G d„ de«      Z) G d„ dejN                  «      Z* G d„ de«      Z+ G d„ d e«      Z, G d!„ d"e«      Z- G d#„ d$e«      Z. G d%„ d&ejN                  «      Z/e G d'„ d(e«      «       Z0e G d)„ d*e0«      «       Z1 G d+„ d,e«      Z2 G d-„ d.e«      Z3g d/¢Z4y)0zPyTorch SEW model.é    N)Únné   )Úinitialization)ÚACT2FN)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚBaseModelOutput)ÚPreTrainedModel)Úauto_docstring)Úis_flash_attention_requestedé   )ÚWav2Vec2AttentionÚWav2Vec2EncoderLayerÚWav2Vec2FeatureEncoderÚWav2Vec2FeedForwardÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2GroupNormConvLayerÚWav2Vec2LayerNormConvLayerÚWav2Vec2NoLayerNormConvLayerÚWav2Vec2SamePadLayerÚ_compute_mask_indicesé   )Ú	SEWConfigc                   ó   — e Zd Zy)ÚSEWNoLayerNormConvLayerN©Ú__name__Ú
__module__Ú__qualname__© ó    ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/sew/modular_sew.pyr   r   0   ó   „ Ør"   r   c                   ó   — e Zd Zy)ÚSEWLayerNormConvLayerNr   r!   r"   r#   r&   r&   4   r$   r"   r&   c                   ó   — e Zd Zy)ÚSEWGroupNormConvLayerNr   r!   r"   r#   r(   r(   8   r$   r"   r(   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWPositionalConvEmbeddingc                 ó¼  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  dz  |j                  |j                  ¬«      | _        t        j                  j                  }t        t        j                  j                  d«      r$t        j                  j                  j                  }t        «       �r(dd l}|j                  j!                  | j                  j"                  d¬«      5   || j                  dd¬«      | _        d d d «       t        | j                  d«      rU| j                  j                  j"                  j$                  }| j                  j                  j"                  j&                  }n,| j                  j(                  }| j                  j*                  }|j                  j-                  | |«       |j                  j-                  | |«       n || j                  dd¬«      | _        t/        |j
                  «      | _        t2        |j4                     | _        y # 1 sw Y   �Œ'xY w)	Nr   )Úkernel_sizeÚpaddingÚgroupsÚstrideÚweight_normr   ©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__r   ÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚsqueeze_factorÚconvÚutilsr0   Úhasattrr6   r   Ú	deepspeedÚzeroÚGatheredParametersr3   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚSEWSamePadLayerr-   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr0   rA   rF   rG   Ú	__class__s         €r#   r8   z#SEWPositionalConvEmbedding.__init__=   s®  ø€ Ü‰ÑÔÜ—I‘IØ×ÑØ×ÑØ×6Ñ6Ø×2Ñ2°aÑ7Ø×7Ñ7Ø×(Ñ(ô
ˆŒ	ô —h‘h×*Ñ*ˆÜ”2—8‘8×,Ñ,¨mÔ<ÜŸ(™(×3Ñ3×?Ñ?ˆKä%Õ'Ûà—‘×2Ñ2°4·9±9×3CÑ3CÐSTÐ2ÓUñ IÙ'¨¯	©	¸ÀaÔH�”	÷Iä�t—y‘yÐ"4Ô5ØŸ9™9×5Ñ5×<Ñ<×FÑF�ØŸ9™9×5Ñ5×<Ñ<×FÑF‘àŸ9™9×-Ñ-�ØŸ9™9×-Ñ-�Ø�N‰N×6Ñ6°t¸XÔFØ�N‰N×6Ñ6°t¸XÕFá# D§I¡I°HÀ!ÔDˆDŒIä& v×'EÑ'EÓFˆŒÜ  ×!?Ñ!?Ñ@ˆ�÷Iñ Iús   ÄIÉIc                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S ©N)r>   r-   rK   )rL   Úhidden_statess     r#   Úforwardz"SEWPositionalConvEmbedding.forward_   s2   € ØŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆàÐr"   ©r   r   r    r8   rR   Ú__classcell__©rN   s   @r#   r*   r*   <   s   ø„ ô AöDr"   r*   c                   ó   — e Zd Zy)rI   Nr   r!   r"   r#   rI   rI   g   r$   r"   rI   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSEWUpsamplingc                 óì   •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  z  «      | _        t        |j                     | _	        |j
                  | _        y rP   )
r7   r8   r   ÚLinearr:   r=   Ú
projectionr   rJ   rK   ©rL   rM   rN   s     €r#   r8   zSEWUpsampling.__init__l   sW   ø€ Ü‰ÑÔÜŸ)™) F×$6Ñ$6¸×8JÑ8JÈV×MbÑMbÑ8bÓcˆŒÜ  ×!?Ñ!?Ñ@ˆŒØ$×3Ñ3ˆÕr"   c                 ó.  — | j                  |«      }| j                  |«      }| j                  dkD  rc|j                  «       \  }}}|| j                  z  }|| j                  z  }|j	                  ||| j                  |«      }|j	                  |||«      }|S )Nr   )r[   rK   r=   ÚsizeÚreshape)rL   rQ   ÚbszÚsrc_lenÚsrc_embed_dimÚtgt_lenÚtgt_embed_dims          r#   rR   zSEWUpsampling.forwardr   s˜   € ØŸ™¨Ó6ˆØŸ™¨Ó6ˆà×Ñ Ò"à*7×*<Ñ*<Ó*>Ñ'ˆC�˜-Ø × 3Ñ 3Ñ3ˆGØ)¨T×-@Ñ-@Ñ@ˆMØ)×1Ñ1°#°wÀ×@SÑ@SÐUbÓcˆMØ)×1Ñ1°#°wÀÓNˆMàÐr"   rS   rU   s   @r#   rX   rX   k   s   ø„ ô4ör"   rX   c                   ó   — e Zd Zy)ÚSEWFeatureEncoderNr   r!   r"   r#   rf   rf   �   r$   r"   rf   c                   ó   — e Zd Zy)ÚSEWAttentionNr   r!   r"   r#   rh   rh   …   r$   r"   rh   c                   ó   — e Zd Zy)ÚSEWFeedForwardNr   r!   r"   r#   rj   rj   ‰   r$   r"   rj   c                   ó   — e Zd Zy)ÚSEWEncoderLayerNr   r!   r"   r#   rl   rl   �   r$   r"   rl   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú
SEWEncoderc                 ó  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _        t        j                   t#        |j$                  «      D �cg c]  }t'        |«      ‘Œ c}«      | _        t+        |«      | _        d| _        y c c}w )N©ÚepsF)r7   r8   rM   r*   Úpos_conv_embedr   Ú	AvgPool1dr=   ÚpoolÚ	LayerNormr:   Úlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropoutÚdropoutÚ
ModuleListÚrangeÚnum_hidden_layersrl   ÚlayersrX   ÚupsampleÚgradient_checkpointing)rL   rM   Ú_rN   s      €r#   r8   zSEWEncoder.__init__’   s¹   ø€ Ü‰ÑÔØˆŒÜ8¸Ó@ˆÔÜ—L‘L ×!6Ñ!6¸×8MÑ8MÓNˆŒ	ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÄeÈF×LdÑLdÓFeÖ$fÀ¤_°VÕ%<Ò$fÓgˆŒÜ% fÓ-ˆŒØ&+ˆÕ#ùò %gs   ÃDc           	      ó  — |rdnd }|rdnd }|��5|j                  d«      j                  dd|j                  d   «      }t        | j                  «      rd|| <   |�d|v r|nd }nàd|| <   t
        j                  j                  |j                  «       j                  d«      | j                  j                  | j                  j                  ¬«      j                  d«      j                  «       }d|d d …d d d d …f   j                  |j                  ¬	«      z
  }|t        j                  |j                  «      j                   z  }|j                  d   }	|j#                  dd«      }| j%                  |«      }
| j'                  |«      }t!        |
j)                  d«      |j)                  d«      «      }|d
d |…f   |
d
d |…f   z   }|j#                  dd«      }| j+                  |«      }| j-                  |«      }t/        «       xs t1        | «      }| j2                  D ]j  }|r||fz   }t        j4                  g «      }| j6                  xr || j                  j8                  k  }|r|r ||||¬«      }|d   }|rd}|sŒb|d   fz   }Œl |r||fz   }| j;                  |«      }|j                  d   |	k  r4t
        j                  j=                  |ddd|	|j                  d   z
  f«      }|st?        d„ |||fD «       «      S tA        |||¬«      S )Nr!   éÿÿÿÿr   r   ç        r   )r,   r/   g      ð?)Údtype.)Úattention_maskÚoutput_attentions©NNc              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrP   r!   )Ú.0Úvs     r#   ú	<genexpr>z%SEWEncoder.forward.<locals>.<genexpr>ï   s   è ø€ Òm˜qÐ_`Ñ_lœÑmùs   ‚Š©Úlast_hidden_staterQ   Ú
attentions)!Ú	unsqueezeÚrepeatÚshaper   rM   r   Ú
functionalÚ
max_pool1dÚfloatr=   ÚsqueezeÚlongÚtor…   ÚtorchÚfinfoÚminÚ	transposerr   rt   r^   rw   rz   r   r   r~   ÚrandÚtrainingÚ	layerdropr   ÚpadÚtupler	   )rL   rQ   r†   r‡   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚn_input_timestepsÚposition_embeddingsÚpooled_hidden_statesÚ
min_lengthÚsynced_gpusÚlayerÚdropout_probabilityÚskip_the_layerÚlayer_outputss                     r#   rR   zSEWEncoder.forward�   s<  € ñ #7™B¸DÐÙ$5™b¸4ÐàÑ%Ø$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ü+¨D¯K©KÔ8à8;�Ð4Ð4Ñ5à4BÐ4NÐSTÐXfÑSf¡Ðmq‘ð 9<�Ð4Ð4Ñ5ô
 —M‘M×,Ñ,Ø&×,Ñ,Ó.×8Ñ8¸Ó;Ø$(§K¡K×$>Ñ$>Ø#Ÿ{™{×9Ñ9ð -ó ÷
 ‘W˜Q“Zß‘T“Vð ð "% ~²a¸¸tÂQÐ6FÑ'G×'JÑ'JÐQ^×QdÑQdÐ'JÓ'eÑ!e�Ø!/´%·+±+¸m×>QÑ>QÓ2R×2VÑ2VÑ!V�à)×/Ñ/°Ñ2Ðà%×/Ñ/°°1Ó5ˆØ"×1Ñ1°-Ó@ÐØ#Ÿy™y¨Ó7ÐÜÐ,×1Ñ1°"Ó5Ð7K×7PÑ7PÐQSÓ7TÓUˆ
Ø,¨S°+°:°+Ð-=Ñ>ÐATÐUXÐZeÐ[eÐZeÐUeÑAfÑfˆØ%×/Ñ/°°1Ó5ˆàŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6Rˆà—[‘[ò 	PˆEÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]ÒZÐ/BÀTÇ[Á[×EZÑEZÑ/ZˆNÙ!¡[á %Ø!°.ÐTeô!�ð !.¨aÑ 0�áØ ,�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð'	Pñ*  Ø 1°]Ð4DÑ DÐàŸ™ mÓ4ˆØ×Ñ˜qÑ!Ð$5Ò5ÜŸM™M×-Ñ-¨m¸aÀÀAÐGXÐ[h×[nÑ[nÐopÑ[qÑGqÐ=rÓsˆMáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r"   )NFFTrS   rU   s   @r#   rn   rn   ‘   s   ø„ ô	,ð ØØ"Ø÷W
r"   rn   c                   ó¸   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej                  «       ˆ fd„«       Zdej                  ez  fd	„Zd
edej                  fd„Zˆ xZS )ÚSEWPreTrainedModelrM   ÚsewÚinput_valuesÚaudioTFc           
      óÈ  •— t         ‰| �  |«       t        |t        «      ržt	        j
                  |j                  j                  ddt        j                  d|j                  j                  d   |j                  j                  z  z  «      z  ¬«       t	        j                  |j                  j                  d«       yt        |t        j                  «      ròt!        «       rÈddl}t%        |d«      rht%        |d«      r\|j&                  j)                  |j*                  |j,                  gd¬«      5  t	        j.                  |j                  «       ddd«       y|j&                  j)                  |j                  d¬«      5  t	        j.                  |j                  «       ddd«       yt	        j.                  |j                  «       yy# 1 sw Y   yxY w# 1 sw Y   yxY w)	zInitialize the weightsr   r   r   )ÚmeanÚstdNrG   rF   r1   )r7   Ú_init_weightsÚ
isinstancer*   ÚinitÚnormal_r>   r3   ÚmathÚsqrtr,   Úin_channelsÚ	constant_Úbiasr   r9   r   rA   r@   rB   rC   rG   rF   Úkaiming_normal_)rL   ÚmodulerA   rN   s      €r#   r¸   z SEWPreTrainedModel._init_weights  se  ø€ ô 	‰Ñ˜fÔ%Ü�fÔ8Ô9Ü�L‰LØ—‘×"Ñ"ØØœŸ	™	 ! v§{¡{×'>Ñ'>¸qÑ'AÀFÇKÁK×D[ÑD[Ñ'[Ñ"\Ó]Ñ]õô
 �N‰N˜6Ÿ;™;×+Ñ+¨QÕ/Ü˜¤§	¡	Ô*Ü)Ô+Û ä˜6 :Ô.´7¸6À:Ô3NØ"Ÿ™×:Ñ:¸F¿O¹OÈVÏ_É_Ð;]ÐmnÐ:Óoñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ð #Ÿ™×:Ñ:¸6¿=¹=ÐXYÐ:ÓZñ <Ü×,Ñ,¨V¯]©]Ô;÷<ð <ô ×$Ñ$ V§]¡]Õ3ð +÷
<ð <ú÷<ð <ús   Ä2 GÆ GÇGÇG!Úinput_lengthsc                 ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                 ó>   — t        j                  | |z
  |d¬«      dz   S )NÚfloor)Úrounding_moder   )r™   Údiv)Úinput_lengthr,   r/   s      r#   Ú_conv_out_lengthzMSEWPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length  s"   € ô —9‘9˜\¨KÑ7¸ÈwÔWÐZ[Ñ[Ð[r"   )ÚziprM   Úconv_kernelÚconv_stride)rL   rÃ   rÊ   r,   r/   s        r#   Ú _get_feat_extract_output_lengthsz3SEWPreTrainedModel._get_feat_extract_output_lengths  sQ   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð Ðr"   Úfeature_vector_lengthr†   c                 ó  — | j                  |j                  d«      «      j                  t        j                  «      }t        j
                  ||j                  ¬«      }|j                  d«      |j                  d«      k  S )Nrƒ   )Údevicer   r   )rÎ   Úsumr˜   r™   r—   ÚarangerÑ   r�   )rL   rÏ   r†   Úoutput_lengthsÚattention_idss        r#   Ú"_get_feature_vector_attention_maskz5SEWPreTrainedModel._get_feature_vector_attention_mask)  sj   € Ø×>Ñ>¸~×?QÑ?QÐRTÓ?UÓV×YÑYÔZ_×ZdÑZdÓeˆô Ÿ™Ð%:À>×CXÑCXÔYˆØ×&Ñ& qÓ)¨N×,DÑ,DÀQÓ,GÑGÐGr"   )r   r   r    r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr™   Úno_gradr¸   Ú
LongTensorÚintrÎ   rÖ   rT   rU   s   @r#   r±   r±   ÷   s€   ø… àÓØÐØ$€OØÐØ&*Ð#ØÐØ€NØÐà€U‡]�]ƒ_ó4ó ð4ð.¸e×>NÑ>NÐQTÑ>Tó ðHÈð HÐ]b×]mÑ]m÷ Hr"   r±   c                   ó&  ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  fd„Ze		 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  deez  fd„«       Zˆ xZS )ÚSEWModelrM   c                 óÄ  •— t         ‰| �  |«       || _        t        |«      | _        t        j                  |j                  d   |j                  ¬«      | _	        |j                  d   |j                  k7  | _        | j                  r2t        j                  |j                  d   |j                  «      | _        t        j                  |j                  «      | _        |j"                  dkD  s|j$                  dkD  rEt        j&                  t)        j*                  |j                  «      j-                  «       «      | _        t1        |«      | _        | j5                  «        y )Nrƒ   rp   r„   )r7   r8   rM   rf   Úfeature_extractorr   ru   Úconv_dimrv   rw   r:   Úproject_featuresrZ   Úfeature_projectionrx   Úfeat_proj_dropoutÚfeature_dropoutÚmask_time_probÚmask_feature_probÚ	Parameterr™   ÚTensorÚuniform_Úmasked_spec_embedrn   ÚencoderÚ	post_initr\   s     €r#   r8   zSEWModel.__init__4  sÿ   ø€ Ü‰Ñ˜Ô ØˆŒÜ!2°6Ó!:ˆÔÜŸ,™, v§¡°rÑ':À×@UÑ@UÔVˆŒà &§¡°Ñ 3°v×7IÑ7IÑ IˆÔØ× Ò Ü&(§i¡i°·±ÀÑ0CÀV×EWÑEWÓ&XˆDÔ#Ü!Ÿz™z¨&×*BÑ*BÓCˆÔà× Ñ  3Ò&¨&×*BÑ*BÀSÒ*HÜ%'§\¡\´%·,±,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"ä! &Ó)ˆŒð 	�‰Õr"   NrQ   Úmask_time_indicesr†   c                 óÎ  — t        | j                  dd«      s|S |j                  «       \  }}}|�)| j                  j	                  |j
                  «      ||<   nË| j                  j                  dkD  r²| j                  r¦t        ||f| j                  j                  | j                  j                  || j                  j                  ¬«      }t        j                  ||j                  t        j                  ¬«      }| j                  j	                  |j
                  «      ||<   | j                  j                  dkD  r¨| j                  rœt        ||f| j                  j                  | j                  j                   | j                  j"                  ¬«      }t        j                  ||j                  t        j                  ¬«      }|dd…df   j%                  d|d«      }d||<   |S )	z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        Úapply_spec_augmentTNr   )Ú	mask_probÚmask_lengthr†   Ú	min_masks)rÑ   r…   )rö   r÷   rø   rƒ   )ÚgetattrrM   r^   rð   r˜   r…   rë   rž   r   Úmask_time_lengthÚmask_time_min_masksr™   ÚtensorrÑ   Úboolrì   Úmask_feature_lengthÚmask_feature_min_masksÚexpand)rL   rQ   ró   r†   Ú
batch_sizeÚsequence_lengthr:   Úmask_feature_indicess           r#   Ú_mask_hidden_stateszSEWModel._mask_hidden_statesH  sš  € ô �t—{‘{Ð$8¸$Ô?Ø Ð ð 4A×3EÑ3EÓ3GÑ0ˆ
�O [àÐ(à/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ò,Ø�[‰[×'Ñ'¨!Ò+°·²Ü 5Ø˜_Ð-ØŸ+™+×4Ñ4Ø ŸK™K×8Ñ8Ø-ØŸ+™+×9Ñ9ô!Ðô !&§¡Ð->À}×G[ÑG[Ôch×cmÑcmÔ nÐØ/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÐ+Ñ,à�;‰;×(Ñ(¨1Ò,°·²ä#8Ø˜[Ð)ØŸ+™+×7Ñ7Ø ŸK™K×;Ñ;ØŸ+™+×<Ñ<ô	$Ð ô $)§<¡<Ð0DÈ]×MaÑMaÔin×isÑisÔ#tÐ Ø#7º¸4¸Ñ#@×#GÑ#GÈÈOÐ]_Ó#`Ð Ø23ˆMÐ.Ñ/àÐr"   r³   r‡   r¢   r£   Úreturnc                 óZ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      }|j                  dd«      }| j                  |«      }| j                  r| j                  |«      }| j                  |«      }	|�| j                  |	j                  d   |«      }| j                  |	|¬«      }	| j                  |	||||¬«      }
|
d   }	|s	|	f|
dd z   S t        |	|
j                  |
j                   ¬«      S )a/  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        Nr   r   )ró   )r†   r‡   r¢   r£   r   r�   )rM   r‡   r¢   r£   rå   rœ   rw   rç   rè   rê   rÖ   r’   r  rñ   r	   rQ   r�   )rL   r³   r†   ró   r‡   r¢   r£   ÚkwargsÚextract_featuresrQ   Úencoder_outputss              r#   rR   zSEWModel.forwardv  sU  € ð  2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐØŸ?™?Ð+;Ó<Ðà× Ò Ø#×6Ñ6Ð7GÓHÐØ×,Ñ,Ð-=Ó>ˆàÐ%à!×DÑDÀ]×EXÑEXÐYZÑE[Ð]kÓlˆNà×0Ñ0°ÐRcÐ0ÓdˆàŸ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Ñ*ˆáØ!Ð# o°a°bÐ&9Ñ9Ð9äØ+Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r"   rˆ   )NNNNN)r   r   r    r   r8   r™   ÚFloatTensorrà   r  r   rî   rý   r¡   r	   rR   rT   rU   s   @r#   rã   rã   2  sì   ø„ ð˜yõ ð. 7;Ø26ñ	,à×(Ñ(ð,ð !×,Ñ,¨tÑ3ð,ð ×(Ñ(¨4Ñ/ó	,ð\ ð /3Ø6:Ø)-Ø,0Ø#'ñ4
à—l‘l TÑ)ð4
ð Ÿ™ tÑ+ð4
ð !×,Ñ,¨tÑ3ð	4
ð
   $™;ð4
ð # T™kð4
ð ˜D‘[ð4
ð 
�Ñ	 ò4
ó ô4
r"   rã   c                   ó   — e Zd Zy)Ú	SEWForCTCNr   r!   r"   r#   r  r  ®  r$   r"   r  c                   ó   — e Zd Zy)ÚSEWForSequenceClassificationNr   r!   r"   r#   r  r  ²  r$   r"   r  )r  r  rã   r±   )5Ú__doc__r¼   r™   r   Ú r   rº   Úactivationsr   Úintegrations.deepspeedr   Úintegrations.fsdpr   Úmodeling_outputsr	   Úmodeling_utilsr
   r?   r   Úutils.genericr   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   r   r   Úconfiguration_sewr   Ú_HIDDEN_STATES_START_POSITIONr   r&   r(   ÚModuler*   rI   rX   rf   rh   rj   rl   rn   r±   rã   r  r  Ú__all__r!   r"   r#   ú<module>r     sH  ðñ ã ã Ý å &Ý !Ý @Ý 7Ý /Ý -Ý #Ý 9÷÷ ÷ ñ õ )ð !"Ð ô	Ð:ô 	ô	Ð6ô 	ô	Ð6ô 	ô( §¡ô (ôV	Ð*ô 	ô�B—I‘Iô ô,	Ð.ô 	ô	Ð$ô 	ô	Ð(ô 	ô	Ð*ô 	ôc
�—‘ô c
ðL ô7H˜ó 7Hó ð7Hðt ôx
Ð!ó x
ó ðx
ôv	�ô 	ô	Ð#Dô 	ò Z�r"   