Ë
    ÜÓ^j†ˆ  ã                   ó2  — d dl mZ d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8m9Z9 ddl:m;Z;m<Z<  e,jz                  e>«      Z? e*d¬«      e G d„ de«      «       «       Z@ e*d¬«      e G d„ d e«      «       «       ZA G d!„ d"ej„                  «      ZC G d#„ d$ej„                  «      ZD G d%„ d&e4«      ZE G d'„ d(e3«      ZF G d)„ d*e7«      ZG G d+„ d,e«      ZHe* G d-„ d.e%«      «       ZI G d/„ d0eI«      ZJ G d1„ d2e8«      ZK G d3„ d4e;«      ZL e*d5¬«       G d6„ d7eIe«      «       ZMg d8¢ZNy)9é    )ÚCallable)Ú	dataclassN)Ústrict)ÚCrossEntropyLossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚPreTrainedConfig)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGlmAttentionÚGlmRotaryEmbeddingÚapply_rotary_pos_emb)ÚLlamaDecoderLayerÚ
LlamaModelÚeager_attention_forward)ÚWhisperModelÚshift_tokens_rightzUsefulSensors/moonshine-tiny)Ú
checkpointc                   óÎ  ‡ — e Zd ZU dZdZdgZdddddœZd	Zee	d
<   dZ
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d <   dZeez  dz  e	d!<   dZee	d"<   d#Z ee	d$<   d%Z!eez  e	d&<   dZ"edz  e	d'<   d(Z#ee$e   z  dz  e	d)<   dZ%edz  e	d*<   dZ&ee	d+<   ˆ fd,„Z'ˆ xZ(S )-ÚMoonshineConfiga‹	  
    encoder_num_key_value_heads (`int`, *optional*):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `encoder_num_key_value_heads=encoder_num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `encoder_num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `num_attention_heads`.
    decoder_num_key_value_heads (`int`, *optional*):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `decoder_num_key_value_heads=decoder_num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `decoder_num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `decoder_num_attention_heads`.
    pad_head_dim_to_multiple_of (`int`, *optional*):
        Pad head dimension in encoder and decoder to the next multiple of this value. Necessary for using certain
        optimized attention implementations.
    encoder_hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
        The non-linear activation function (function or string) in the encoder.
    decoder_hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.

    Example:

    ```python
    >>> from transformers import MoonshineModel, MoonshineConfig

    >>> # Initializing a Moonshine style configuration
    >>> configuration = MoonshineConfig().from_pretrained("UsefulSensors/moonshine-tiny")

    >>> # Initializing a model from the configuration
    >>> model = MoonshineModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú	moonshineÚpast_key_valuesÚdecoder_num_key_value_headsÚdecoder_num_attention_headsÚdecoder_num_hidden_layersÚdecoder_hidden_act)Únum_key_value_headsÚnum_attention_headsÚnum_hidden_layersÚ
hidden_acti €  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Úencoder_num_hidden_layersé   Úencoder_num_attention_headsNÚencoder_num_key_value_headsÚpad_head_dim_to_multiple_ofÚgeluÚencoder_hidden_actÚsilui   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeé   Údecoder_start_token_idTÚ	use_cacheÚrope_parametersÚis_encoder_decoderFÚattention_biasç        Úattention_dropoutÚbos_token_idr"   Úeos_token_idÚpad_token_idÚtie_word_embeddingsc                 ó¼   •— | j                   €| j                  | _         | j                  €| j                  | _        |j	                  dd«       t        ‰| �  di |¤Ž y )NÚpartial_rotary_factorgÍÌÌÌÌÌì?© )r?   r>   r0   r1   Ú
setdefaultÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/moonshine/modular_moonshine.pyrW   zMoonshineConfig.__post_init__€   sX   ø€ Ø×+Ñ+Ð3Ø/3×/OÑ/OˆDÔ,à×+Ñ+Ð3Ø/3×/OÑ/OˆDÔ,à×ÑÐ1°3Ô7Ü‰ÑÑ' Ó'ó    ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr8   ÚintÚ__annotations__r9   r:   r<   r2   r>   r1   r?   r0   r@   rB   Ústrr3   rD   rE   ÚfloatrG   rH   ÚboolrI   r   ÚdictrJ   rK   rM   rN   rO   ÚlistrP   rQ   rW   Ú__classcell__©rZ   s   @r[   r-   r-   3   sg  ø… ñ&ðP €JØ#4Ð"5Ðà<Ø<Ø8Ø*ñ	€Mð €J�ÓØ€K�ÓØ!Ð�sÓ!Ø%&Ð˜sÓ&Ø%&Ð˜sÓ&Ø'(Ð Ó(Ø'(Ð Ó(Ø.2Ð  t¡Ó2Ø.2Ð  t¡Ó2Ø.2Ð  t¡Ó2Ø$Ð˜Ó$Ø$Ð˜Ó$Ø#&Ð˜SÓ&Ø#Ð�uÓ#Ø"#Ð˜CÓ#Ø€IˆtÓØ48€O�^ dÑ*¨TÑ1Ó8Ø#Ð˜Ó#Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø#€L�#˜‘*Ó#Ø $Ð˜Ó$÷(ð (r\   r-   z™
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )Úcustom_introc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚMoonshineEncoderModelOutputa–  
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
        sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    NÚattention_mask)r]   r^   r_   r`   rp   ÚtorchÚTensorre   rT   r\   r[   ro   ro   ‹   s   … ñð +/€N�E—L‘L 4Ñ'Ô.r\   ro   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMoonshineEncoderMLPc                 ó  •— t         ‰| �  «        || _        t        |   | _        t        j                  |j                  |j                  «      | _	        t        j                  |j                  |j                  «      | _
        y ©N©rV   Ú__init__Úconfigr   Úactivation_fnÚnnÚLinearr9   r:   Úfc1Úfc2©rX   ry   r7   rZ   s      €r[   rx   zMoonshineEncoderMLP.__init__Ÿ   s^   ø€ Ü‰ÑÔØˆŒÜ# JÑ/ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r\   Úhidden_statesÚreturnc                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rv   )r}   rz   r~   )rX   r€   s     r[   ÚforwardzMoonshineEncoderMLP.forward¦   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr\   ©r]   r^   r_   rx   rq   rr   rƒ   rk   rl   s   @r[   rt   rt   ž   s$   ø„ ôKð U§\¡\ð °e·l±l÷ r\   rt   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMoonshineDecoderMLPc                 ó  •— t         ‰| �  «        || _        t        |   | _        t        j                  |j                  |j                  dz  «      | _	        t        j                  |j                  |j                  «      | _
        y )Nr"   rw   r   s      €r[   rx   zMoonshineDecoderMLP.__init__®   sc   ø€ Ü‰ÑÔØˆŒÜ# JÑ/ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÈAÑ1MÓNˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r\   r€   r�   c                 óž   — | j                  |«      }|j                  dd¬«      \  }}| j                  |«      |z  }| j                  |«      }|S )Nr"   éÿÿÿÿ)Údim)r}   Úchunkrz   r~   )rX   r€   Úgates      r[   rƒ   zMoonshineDecoderMLP.forwardµ   sS   € ØŸ™ Ó/ˆØ+×1Ñ1°!¸Ð1Ó<Ñˆ�tØ×*Ñ*¨4Ó0°=Ñ@ˆØŸ™ Ó/ˆØÐr\   r„   rl   s   @r[   r†   r†   ­   s$   ø„ ôKð U§\¡\ð °e·l±l÷ r\   r†   c                   ó   — e Zd Zy)ÚMoonshineRotaryEmbeddingN)r]   r^   r_   rT   r\   r[   rŽ   rŽ   ½   s   „ Ør\   rŽ   c                   óH  ‡ — e Zd Zdededededef
ˆ fd„Z	 	 	 	 ddej                  d	e	ej                  ej                  f   dz  d
ej                  dz  de
dz  dej                  dz  dee   de	ej                  ej                  dz  e	ej                     dz  f   fd„Zˆ xZS )ÚMoonshineAttentionry   Ú	layer_idxÚ	is_causalr5   r4   c                 ón  •— |j                  ||dœ«       t        ‰| �	  ||«       || _        t	        |d|j
                  |j                  z  «      | _        | j                  j                  �C| j                  j                  }|| j                  |z   dz
  |z  z  }|| j                  z
  | _
        y d| _
        y )N)r5   r4   Úhead_dimrF   r   )ÚupdaterV   rx   r’   Úgetattrr9   r5   r”   ry   r@   Úhead_dim_padding)	rX   ry   r‘   r’   r5   r4   Útarget_multipleÚtarget_head_dimrZ   s	           €r[   rx   zMoonshineAttention.__init__Â   s¨   ø€ ð 	�‰Ð.AÐZmÑnÔoÜ‰Ñ˜ Ô+Ø"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒð �;‰;×2Ñ2Ð>Ø"Ÿk™k×EÑEˆOØ-°$·-±-À/Ñ2QÐTUÑ2UÐZiÑ1iÑjˆOØ$3°d·m±mÑ$CˆDÕ!à$%ˆDÕ!r\   Nr€   Úposition_embeddingsrp   r/   Úkey_value_statesrY   r�   c                 ó:  — |j                   d d \  }}| j                  |«      j                  ||| j                  j                  | j
                  «      j                  dd«      }	|d u}
|�Y|j                  j                  | j                  «      }|
r&d|j                  | j                  <   |j                  }n|j                  }|�|n|}|
rK|rIrG|j                  | j                     j                  }|j                  | j                     j                  }nÆ| j                  |«      j                  |d| j                  j                  | j
                  «      j                  dd«      }| j!                  |«      j                  |d| j                  j                  | j
                  «      j                  dd«      }|
r"|� |j#                  ||| j                  «      \  }}|
s8|\  }}t%        |	|||«      \  }	}|� |j#                  ||| j                  «      \  }}t'        j(                  | j                  j*                  t,        «      }| j.                  xr |d u xr |dkD  }| j0                  dkD  r¢t2        j4                  j6                  j9                  |	d| j0                  f«      }	t2        j4                  j6                  j9                  |d| j0                  f«      }t2        j4                  j6                  j9                  |d| j0                  f«      } || |	|||f| j:                  sdn| j<                  | j>                  |dœ|¤Ž\  }}| j0                  dkD  r|dd | j0                   …f   }|jA                  ||d«      jC                  «       }| jE                  |«      }||fS )	Nr‰   rF   r"   Tr   rL   )ÚdropoutÚscalingr’   .)#ÚshapeÚq_projÚviewry   r4   r”   Ú	transposeÚ
is_updatedÚgetr‘   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesÚk_projÚv_projr•   r%   r   Úget_interfaceÚ_attn_implementationr(   r’   r—   rq   r{   Ú
functionalÚpadÚtrainingrM   rž   ÚreshapeÚ
contiguousÚo_proj)rX   r€   rš   rp   r/   r›   rY   ÚbszÚq_lenÚquery_statesÚis_cross_attentionr£   Úcurrent_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfacer’   Úattn_outputÚattn_weightss                        r[   rƒ   zMoonshineAttention.forward×   s]  € ð #×(Ñ(¨¨"Ð-‰
ˆˆUð �K‰K˜Ó&×+Ñ+¨C°¸¿¹×8WÑ8WÐY]×YfÑYfÓg×qÑqÐrsÐuvÓwð 	ð .°TÐ9ÐØÐ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÙ!à=A�×*Ñ*¨4¯>©>Ñ:Ø"1×"GÑ"G‘à"1×"FÑ"F�ð .>Ð-IÑ)È}ˆÙ¡/±jØ(×/Ñ/°·±Ñ?×DÑDˆJØ*×1Ñ1°$·.±.ÑA×HÑH‰Lð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ñ
 " oÐ&AØ+:×+AÑ+AÀ*ÈlÐ\`×\jÑ\jÓ+kÑ(�
˜Lá!Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ*Ø+:×+AÑ+AÀ*ÈlÐ\`×\jÑ\jÓ+kÑ(�
˜Lä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðð —N‘NÒK ~¸Ð'=ÒKÀ%È!Á)ˆ	à× Ñ  1Ò$Ü Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLÜŸ™×,Ñ,×0Ñ0°¸aÀ×AVÑAVÐ=WÓXˆJÜ Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLá$7ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØñ
%
ð ñ
%
Ñ!ˆ�\ð × Ñ  1Ò$Ø% cÐ+C¨d×.CÑ.CÐ-CÐ+CÐ&CÑDˆKà!×)Ñ)¨#¨u°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r\   )NNNN)r]   r^   r_   r-   rd   rh   rx   rq   rr   Útupler	   r   r   rƒ   rk   rl   s   @r[   r�   r�   Á   s÷   ø„ ð&àð&ð ð&ð ð	&ð
 !ð&ð !õ&ð0 IMØ.2Ø(,Ø04ñO)à—|‘|ðO)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEðO)ð Ÿ™ tÑ+ð	O)ð
  ™ðO)ð  Ÿ,™,¨Ñ-ðO)ð Ð-Ñ.ðO)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷O)r\   r�   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMoonshineEncoderLayerry   r‘   c                 óF  •— t         ‰| �  ||«       t        ||d|j                  |j                  ¬«      | _        t        ||j                  «      | _        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        y )NF©ry   r‘   r’   r5   r4   ©Úbias)rV   rx   r�   r>   r?   Ú	self_attnrt   rB   Úmlpr{   Ú	LayerNormr9   Úinput_layernormÚpost_attention_layernorm©rX   ry   r‘   rZ   s      €r[   rx   zMoonshineEncoderLayer.__init__*  s�   ø€ Ü‰Ñ˜ Ô+ä+ØØØØ &× BÑ BØ &× BÑ Bô
ˆŒô ' v¨v×/HÑ/HÓIˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯©°V×5GÑ5GÈeÔ(TˆÕ%r\   )r]   r^   r_   r-   rd   rx   rk   rl   s   @r[   rÂ   rÂ   )  s   ø„ ðU˜ð U¸3÷ Uñ Ur\   rÂ   c                   óê  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de	dz  de
dz  deej                  ej                  f   dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚMoonshineDecoderLayerNry   r‘   c                 ó  •— t         ‰| �  «        |j                  | _        t        ||d|j                  |j
                  ¬«      | _        t        ||d|j                  |j
                  ¬«      | _        t        ||j                  «      | _
        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        y )NTrÄ   FrÅ   )rV   rx   r9   r�   r5   r4   rÇ   Úencoder_attnr†   r7   rÈ   r{   rÉ   rÊ   rË   Úfinal_layernormrÌ   s      €r[   rx   zMoonshineDecoderLayer.__init__;  sÎ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆŒô /ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆÔô ' v¨v×/@Ñ/@ÓAˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯©°V×5GÑ5GÈeÔ(TˆÔ%Ü!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÕr\   r€   rp   Úencoder_hidden_statesÚencoder_attention_maskÚposition_idsÚencoder_position_idsr/   rH   rš   Úencoder_position_embeddingsrY   r�   c           
      ó&  — |}| j                  |«      } | j                  d||||||	dœ|¤Ž\  }}||z   }|�1|}| j                  |«      }| j                  |||||¬«      \  }}||z   }|}| j	                  |«      }| j                  |«      }||z   }|S )N)r€   rp   rÔ   r/   rH   rš   )r€   r›   rp   r/   rH   rT   )rÊ   rÇ   rË   rÐ   rÑ   rÈ   )rX   r€   rp   rÒ   rÓ   rÔ   rÕ   r/   rH   rš   rÖ   rY   ÚresidualÚ_s                 r[   rƒ   zMoonshineDecoderLayer.forwardS  sÞ   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆà Ð,Ø$ˆHØ ×9Ñ9¸-ÓHˆMØ#×0Ñ0Ø+Ø!6Ø5Ø /Ø#ð  1ó  ÑˆM˜1ð % }Ñ4ˆMà ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐr\   rv   )	NNNNNNFNN)r]   r^   r_   r-   rd   rx   rq   rr   Ú
LongTensorr	   rh   rÀ   r   r   ÚFloatTensorrƒ   rk   rl   s   @r[   rÎ   rÎ   :  sR  ø„ ñL˜ð L¸3À¹:õ Lð6 /3Ø59Ø6:Ø04Ø8<Ø(,Ø!&ØHLØPTñ,à—|‘|ð,ð Ÿ™ tÑ+ð,ð  %Ÿ|™|¨dÑ2ð	,ð
 !&§¡¨tÑ 3ð,ð ×&Ñ&¨Ñ-ð,ð $×.Ñ.°Ñ5ð,ð  ™ð,ð ˜$‘;ð,ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð,ð &+¨5¯<©<¸¿¹Ð+EÑ%FÈÑ%Mð,ð Ð+Ñ,ð,ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷,r\   rÎ   c                   ó\   — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdej                  fd	„Zy
)ÚMoonshinePreTrainedModelry   ÚmodelÚinput_valuesÚaudioTrÂ   rÎ   Úinput_lengthsc                 ó~   — t        |dz
  dz  dz   «      }t        |dz
  dz  dz   «      }t        |dz
  dz  dz   «      }|S )zH
        Computes the output length of the convolutional layers
        é   é@   rF   é   r   r"   )rd   )rX   rá   Úoutput_conv1_lengthÚoutput_conv2_lengthÚoutput_conv3_lengths        r[   Ú _get_feat_extract_output_lengthsz9MoonshinePreTrainedModel._get_feat_extract_output_lengths�  sZ   € ô " =°3Ñ#6¸"Ñ"<¸qÑ"@ÓAÐÜ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÓDÐÜ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÓDÐà"Ð"r\   N)r]   r^   r_   r-   re   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphrq   rÚ   ré   rT   r\   r[   rÝ   rÝ   ‚  sN   … àÓØÐØ$€OØÐØ&*Ð#Ø0Ð2IÐJÐØÐØ€Nà!Ðð#¸e×>NÑ>Nô #r\   rÝ   c                   óà   ‡ — e Zd ZdZdZeedœZdefˆ fd„Z	de
j                  fd„Zde
j                  fd	„Zee	 ddej"                  dej$                  d
z  dee   deez  fd„«       «       Zˆ xZS )ÚMoonshineEncoderz£
    Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

    Args:
        config: MoonshineConfig
    rß   )Ú
attentionsr€   ry   c           	      ób  •— t         ‰| �  |«       || _        |j                  }t	        j
                  d|ddd¬«      | _        t	        j
                  |d|z  dd¬	«      | _        t	        j
                  d|z  |dd¬	«      | _        t	        j                  d|d
¬«      | _
        t	        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t	        j                   |d¬«      | _        t%        |¬«      | _        d| _        | j+                  «        y c c}w )NrF   rã   rä   F)Úkernel_sizeÚstriderÆ   r"   rå   r   )rö   r÷   gñhãˆµøä>)Ú
num_groupsÚnum_channelsÚepsrÅ   ©ry   )rV   rx   ry   r9   r{   ÚConv1dÚconv1Úconv2Úconv3Ú	GroupNormÚ	groupnormÚ
ModuleListÚranger<   rÂ   r§   rÉ   Ú
layer_normrŽ   Ú
rotary_embÚgradient_checkpointingÚ	post_init)rX   ry   Ú	embed_dimÚidxrZ   s       €r[   rx   zMoonshineEncoder.__init__©  sì   ø€ Ü‰Ñ˜Ô ØˆŒØ×&Ñ&ˆ	ä—Y‘Y˜q )¸ÀRÈeÔTˆŒ
Ü—Y‘Y˜y¨!¨i©-ÀQÈqÔQˆŒ
Ü—Y‘Y˜q 9™}¨iÀQÈqÔQˆŒ
ÜŸ™°ÀÐPTÔUˆŒä—m‘mÜ;@À×AaÑAaÓ;bÖc°CÔ" 6¨3Õ/Òcó
ˆŒô Ÿ,™, y°uÔ=ˆŒÜ2¸&ÔAˆŒØ&+ˆÔ#Ø�‰Õùò ds   ÃD,r�   c                 ó   — | j                   S rv   ©rý   ©rX   s    r[   Úget_input_embeddingsz%MoonshineEncoder.get_input_embeddings»  s   € Ø�z‰zÐr\   Úvaluec                 ó   — || _         y rv   r  )rX   r  s     r[   Úset_input_embeddingsz%MoonshineEncoder.set_input_embeddings¾  s	   € Øˆ�
r\   Nrp   rY   c                 óh  — |j                  d«      }t        j                  j                  | j	                  |«      «      }| j                  |«      }t        j                  j                  | j                  |«      «      }t        j                  j                  | j                  |«      «      }|j                  ddd«      }d}|�3| j                  |j                  d   «      }d}|ddd|…f   dd|…f   }|}t        | j                  |||¬«      }t        j                  d|j                  d   |j                   ¬	«      j                  d«      }| j#                  ||¬
«      }	| j$                  D ]  }
 |
|f|||	dœ|¤Ž}Œ | j'                  |«      }t)        ||�|j+                  «       ¬«      S d¬«      S )a.  
        Args:
            input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
                Float values of the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
                `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
                the soundfile library (`pip install soundfile`). To prepare the array into
                `input_values`, the [`AutoFeatureExtractor`] should be used for padding
                and conversion into a tensor of type `torch.FloatTensor`.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        rF   r   r"   Nr‰   i€  .©ry   Úinputs_embedsrp   rÒ   ©Údevice©rÔ   )rp   rÔ   rš   )Úlast_hidden_staterp   )Ú	unsqueezer{   r®   Útanhrý   r  rA   rþ   rÿ   Úpermuteré   rŸ   r   ry   rq   Úaranger  r  r§   r  ro   rd   )rX   rß   rp   rY   r€   Úoutput_attention_maskÚmask_lenÚdownsample_striderÔ   rš   Úencoder_layers              r[   rƒ   zMoonshineEncoder.forwardÁ  s»  € ð. $×-Ñ-¨aÓ0ˆÜŸ™×*Ñ*¨4¯:©:°lÓ+CÓDˆØŸ™ }Ó5ˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆØ%×-Ñ-¨a°°AÓ6ˆð !%ÐØÐ%Ø×<Ñ<¸^×=QÑ=QÐRTÑ=UÓVˆHØ *ÐØ+¨CÑ1DÐ3DÐ1DÐ,DÑEÀcÈ9ÈHÈ9ÀnÑUˆNØ$2Ð!ä2Ø—;‘;Ø'Ø)Ø"/ô	
ˆô —|‘| A }×':Ñ':¸1Ñ'=Àm×FZÑFZÔ[×eÑeÐfgÓhˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà-Ø)Ø$7ñ	ð
 ñ‰Mð	ð Ÿ™¨Ó6ˆä*Ø+Ø:OÐ:[Ð0×4Ñ4Ó6ô
ð 	
àaeô
ð 	
r\   rv   )r]   r^   r_   r`   rë   r�   rÂ   Ú_can_record_outputsr-   rx   r{   ÚModuler  r  r   r!   rq   rÛ   rr   r   r   rÀ   r   rƒ   rk   rl   s   @r[   ró   ró   ›  s«   ø„ ñð %€Oà(Ø.ñÐð
˜õ ð$ b§i¡ió ð¨"¯)©)ó ð  Øð /3ñ<
à×'Ñ'ð<
ð Ÿ™ tÑ+ð<
ð Ð+Ñ,ð	<
ð
 
Ð(Ñ	(ò<
ó ó  ô<
r\   ró   c                   ób  ‡ — e Zd ZdZ eedd¬«      e eedd¬«      dœZdefˆ fd„Z	e
e	 	 	 	 	 	 	 	 ddej                  d	z  d
ej                  d	z  dej                  d	z  ded	z  dej                   d	z  ded	z  dej                   d	z  dej                  d	z  dee   deez  fd„«       «       Zˆ xZS )ÚMoonshineDecoderÚ	input_idsrF   rÇ   )ÚindexÚ
layer_namerÐ   )rô   r€   Úcross_attentionsry   c           	      ó  •— t         ‰| �  |«       t        j                  |j                  d¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _
        y c c}w ©NFrÅ   )rV   rx   r{   rÉ   r9   Únormr  r  r6   rÎ   r§   )rX   ry   r	  rZ   s      €r[   rx   zMoonshineDecoder.__init__
  s[   ø€ Ü‰Ñ˜Ô Ü—L‘L ×!3Ñ!3¸%Ô@ˆŒ	Ü—m‘mÔSXÐY_×YqÑYqÓSrÖ$sÈCÔ%:¸6À3Õ%GÒ$sÓtˆ�ùÒ$ss   ÁA=Nrp   rÔ   r/   r  rH   rÒ   rÓ   rY   r�   c	           
      ó’  — |du |duz  rt        d«      ‚|€| j                  |«      }|r6|€4t        t        | j                  ¬«      t        | j                  ¬«      «      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }t        | j                  ||||¬«      }t        | j                  |||¬«      }|}| j                  ||¬	«      }| j                  D ]  } ||||f|||||d
œ|	¤Ž}Œ | j                  |«      }t!        ||r|¬«      S d¬«      S )a¤  
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)
        Nz:You must specify exactly one of input_ids or inputs_embedsrû   r   rF   r  )ry   r  rp   r/   rÔ   r  r  )rÓ   rÔ   r/   rH   rš   )r  r/   )Ú
ValueErrorÚembed_tokensr   r
   ry   Úget_seq_lengthrq   r  rŸ   r  r  r   r   r  r§   r*  r   )rX   r$  rp   rÔ   r/   r  rH   rÒ   rÓ   rY   Úpast_seen_tokensÚcausal_maskr€   rš   Údecoder_layers                  r[   rƒ   zMoonshineDecoder.forward  s‚  € ð0 ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)ØØØ%ð
ð (>Ø)Ø /Ø#Ø$7ñ
ð ñ
‰Mð	ð Ÿ	™	 -Ó0ˆä8Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r\   )NNNNNNNN)r]   r^   r_   rë   r    r�   rÎ   r   r-   rx   r   r!   rq   rÚ   rr   r	   rÛ   rh   r   r   rÀ   r   rƒ   rk   rl   s   @r[   r#  r#    s2  ø„ Ø!€Oá$Ð%7¸qÈ[ÔYØ.Ù*Ð+=ÀQÐSaÔbñÐðu˜õ uð
  Øð .2Ø.2Ø04Ø(,Ø26Ø!%Ø:>Ø6:ñG
à×#Ñ# dÑ*ðG
ð Ÿ™ tÑ+ðG
ð ×&Ñ&¨Ñ-ð	G
ð
  ™ðG
ð ×(Ñ(¨4Ñ/ðG
ð ˜$‘;ðG
ð  %×0Ñ0°4Ñ7ðG
ð !&§¡¨tÑ 3ðG
ð Ð+Ñ,ðG
ð 
Ð(Ñ	(òG
ó ó  ôG
r\   r#  c                   óN  — e Zd Zd„ Zee	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  de	e	ej                        dz  de
dz  d	e	ej                     dz  d
e	ej                     dz  dedz  dee   defd„«       «       Zy)ÚMoonshineModelc                 ó   — t        d«      ‚)NzNot needed for Moonshine)ÚAttributeErrorr  s    r[   Ú_mask_input_featuresz#MoonshineModel._mask_input_features\  s   € ÜÐ7Ó8Ð8r\   Nrß   rp   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr/   Údecoder_inputs_embedsÚdecoder_position_idsrH   rY   r�   c
                 óT  — |€ | j                   |fd|i|
¤Ž} | j                  d|||j                  |j                  ||||	dœ|
¤Ž}t	        |j                  |j
                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, MoonshineModel
        >>> from datasets import load_dataset

        >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 288]
        ```
        rp   )r$  rp   rÒ   rÓ   r/   r  rÔ   rH   )r  r/   Údecoder_hidden_statesÚdecoder_attentionsr'  Úencoder_last_hidden_staterÒ   Úencoder_attentionsrT   )	ÚencoderÚdecoderr  rp   r   r/   r€   rô   r'  )rX   rß   rp   r7  r8  r9  r/   r:  r;  rH   rY   Údecoder_outputss               r[   rƒ   zMoonshineModel.forward_  sÄ   € ðZ Ð"Ø/;¨t¯|©|¸LÑ/rÐYgÐ/rÐkqÑ/rˆOàEQÀTÇ\Á\ð 
F
Ø'Ø1Ø"1×"CÑ"CØ#2×#AÑ#AØ+Ø/Ø-Øñ
F
ð ñ
F
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r\   )	NNNNNNNNN)r]   r^   r_   r6  r   r   rq   rÛ   rÚ   rÀ   r   rh   r   r   r   rƒ   rT   r\   r[   r3  r3  [  s)  „ ò9ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%ñC
à×'Ñ'¨$Ñ.ðC
ð ×(Ñ(¨4Ñ/ðC
ð !×+Ñ+¨dÑ2ð	C
ð
 !&× 0Ñ 0°4Ñ 7ðC
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðC
ð -¨tÑ3ðC
ð  % U×%6Ñ%6Ñ7¸$Ñ>ðC
ð $ E×$4Ñ$4Ñ5¸Ñ<ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
òC
ó ó ñC
r\   r3  zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                   ó´  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zd„ Zdej                  fd„Z
ee	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deeej                        d	z  ded	z  deej                     d	z  deej                     d	z  ded	z  dej                  d	z  dee   defd„«       «       Zˆ xZS )Ú!MoonshineForConditionalGenerationzproj_out.weightz!model.decoder.embed_tokens.weightry   c                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y r)  )
rV   rx   r3  rÞ   r{   r|   r9   r8   Úproj_outr  )rX   ry   rZ   s     €r[   rx   z*MoonshineForConditionalGeneration.__init__¯  sH   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
ÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÈeÔTˆŒð 	�‰Õr\   c                 ó   — | j                   S rv   ©rG  r  s    r[   Úget_output_embeddingsz7MoonshineForConditionalGeneration.get_output_embeddings·  s   € Ø�}‰}Ðr\   c                 ó   — || _         y rv   rI  )rX   Únew_embeddingss     r[   Úset_output_embeddingsz7MoonshineForConditionalGeneration.set_output_embeddingsº  s	   € Ø&ˆ�r\   r�   c                 ó6   — | j                   j                  «       S rv   )rÞ   r  r  s    r[   r  z6MoonshineForConditionalGeneration.get_input_embeddings½  s   € Ø�z‰z×.Ñ.Ó0Ð0r\   Nrß   rp   r7  r8  r9  r/   r:  r;  rH   ÚlabelsrY   c                 ó.  — |
�9|€7|€5t        |
| j                  j                  | j                  j                  «      } | j                  |f||||||||	dœ|¤Ž}| j                  |j                  «      }d}|
�Ft        «       } ||j                  d| j                  j                  «      |
j                  d«      «      }t        |||j                  |j                  |j                  |j                  |j                  |j                   |j"                  ¬«	      S )a0  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values

        >>> generated_ids = model.generate(input_values, max_new_tokens=100)

        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```N)rp   r7  r9  r8  r/   r:  r;  rH   r‰   )	ÚlossÚlogitsr/   r=  r>  r'  r?  rÒ   r@  )r*   ry   rP   rG   rÞ   rG  r  r   r±   r8   r   r/   r=  r>  r'  r?  rÒ   r@  )rX   rß   rp   r7  r8  r9  r/   r:  r;  rH   rO  rY   ÚoutputsrR  rQ  Úloss_fcts                   r[   rƒ   z)MoonshineForConditionalGeneration.forwardÀ  s  € ðd ÐØ Ð(Ð-BÐ-JÜ$6Ø˜DŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð '1 d§j¡jØð'
à)Ø/Ø+Ø#9Ø+Ø"7Ø!5Øñ'
ð ñ'
ˆð —‘˜w×8Ñ8Ó9ˆàˆØÐÜ'Ó)ˆHÙ˜FŸN™N¨2¨t¯{©{×/EÑ/EÓFÈÏÉÐWYÓHZÓ[ˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r\   )
NNNNNNNNNN)r]   r^   r_   Ú_tied_weights_keysr-   rx   rJ  rM  r{   r!  r  r   r   rq   rÛ   rÚ   rÀ   r   rh   r   r   r   rƒ   rk   rl   s   @r[   rE  rE  §  sr  ø„ ð ,Ð-PÐQÐð˜õ òò'ð1 b§i¡ió 1ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%Ø*.ñS
à×'Ñ'¨$Ñ.ðS
ð ×(Ñ(¨4Ñ/ðS
ð !×+Ñ+¨dÑ2ð	S
ð
 !&× 0Ñ 0°4Ñ 7ðS
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðS
ð -¨tÑ3ðS
ð  % U×%6Ñ%6Ñ7¸$Ñ>ðS
ð $ E×$4Ñ$4Ñ5¸Ñ<ðS
ð ˜$‘;ðS
ð × Ñ  4Ñ'ðS
ð Ð+Ñ,ðS
ð 
òS
ó ó ôS
r\   rE  )r-   r3  rÝ   rE  )OÚcollections.abcr   Údataclassesr   rq   Útorch.nnr{   Úhuggingface_hub.dataclassesr   r   Úactivationsr   Úcache_utilsr	   r
   r   Úconfiguration_utilsr   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr    r!   Úglm.modeling_glmr#   r$   r%   Úllama.modeling_llamar&   r'   r(   Úwhisper.modeling_whisperr)   r*   Ú
get_loggerr]   Úloggerr-   ro   r!  rt   r†   rŽ   r�   rÂ   rÎ   rÝ   ró   r#  r3  rE  Ú__all__rT   r\   r[   ú<module>rn     s©  ðõ %Ý !ã Ý Ý .Ý %å !ß CÑ CÝ 3Ý )ß JÝ BÝ 9÷õ õ 2ß FÝ &ß RÓ RÝ 7ß Eß UÑ Uß YÑ Yß Gð 
ˆ×	Ñ	˜HÓ	%€ñ Ð9Ô:ØôS(Ð&ó S(ó ó ;ðS(ñl ðôð
 ô
/ /ó 
/ó óð
/ô˜"Ÿ)™)ô ô˜"Ÿ)™)ô ô 	Ð1ô 	ôe)˜ô e)ôPUÐ-ô Uô"EÐ6ô EðP ô#˜ó #ó ð#ô0d
Ð/ô d
ôNV
�zô V
ôrI
�\ô I
ñX ðôô
i
Ð(@À/ó i
óð
i
òX�r\   