
    ^jv                       d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZmZmZmZ ddlmZmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ erddlm,Z, ddl-m.Z. ddl/m0Z0  e)jb                  e2      Z3 G d de
jh                        Z5e' G d d             Z6e' G d d             Z7e' G d d             Z8 G d de
jh                        Z9 G d de      Z:y)     )annotationsN)partial)TYPE_CHECKING)	safe_open   )Cache)get_model_conversion_mapping)WeightRenaming$convert_and_load_state_dict_in_model)&LAYER_PATTERN_TO_MASK_FUNCTION_MAPPINGcreate_causal_mask)BaseModelOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)LoadStateDictConfigPreTrainedModel_get_resolved_checkpoint_files)	AutoModel)Unpack)ContextManagersTransformersKwargsauto_docstringcan_return_tuplelogging)log_state_dict_report)MtpCache)PreTrainedConfig)LogitsProcessorListc                  &     e Zd ZdZdZ fdZ xZS )GradientCheckpointingLayera  Base class for layers with gradient checkpointing.

    This class enables gradient checkpointing functionality for a layer. By default, gradient checkpointing is disabled
    (`gradient_checkpointing = False`). When `model.set_gradient_checkpointing()` is called, gradient checkpointing is
    enabled by setting `gradient_checkpointing = True` and assigning a checkpointing function to `_gradient_checkpointing_func`.

    Important:

        When using gradient checkpointing with `use_reentrant=True`, inputs that require gradients (e.g. hidden states)
        must be passed as positional arguments (`*args`) rather than keyword arguments to properly propagate gradients.

        Example:

            ```python
            >>> # Correct - hidden_states passed as positional arg
            >>> out = self.layer(hidden_states, attention_mask=attention_mask)

            >>> # Incorrect - hidden_states passed as keyword arg
            >>> out = self.layer(hidden_states=hidden_states, attention_mask=attention_mask)
            ```
    Fc                   | j                   r| j                  rd}| j                  j                  }d| d}d|v r|d   rd|d<   |dz  }d}d|v r|d   d |d<   |dz  }d}d	|v r|d	   d |d	<   |d
z  }d}d|v r|d   d |d<   |dz  }d}|r)|j	                  d      dz   }t
        j                  |        | j                  t        t        | (  fi |g| S t        | (  |i |S )NFz7Caching is incompatible with gradient checkpointing in z	. Setting	use_cachez `use_cache=False`,Tpast_key_valuez `past_key_value=None`,past_key_valuesz `past_key_values=None`,
layer_pastz `layer_past=None`,,.)gradient_checkpointingtraining	__class____name__rstriploggerwarning_once_gradient_checkpointing_funcr   super__call__)selfargskwargsdo_warn
layer_namemessager+   s         g/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/modeling_layers.pyr2   z#GradientCheckpointingLayer.__call__L   sM   &&4==G00JOPZ|[deGf$)<&+{#00  6)f5E.F.R+/'(44 F*v6G/H/T,0()55v%&*>*J'+|$00 !..-3##G,4444WUW=M5XQW5X`[_``w000    )r,   
__module____qualname____doc__r)   r2   __classcell__r+   s   @r9   r!   r!   3   s    , #"1 "1r:   r!   c                  p     e Zd ZdZ fdZee	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd              Z xZS ) GenericForSequenceClassificationmodelc                8   t         |   |       |j                  | _        t        | | j                  t        j                  |             t        j                  |j                         j                  | j                  d      | _        | j                          y )NFbias)r1   __init__
num_labelssetattrbase_model_prefixr   from_confignnLinearget_text_confighidden_sizescore	post_initr3   configr+   s     r9   rF   z)GenericForSequenceClassification.__init__u   sn      ++d,,i.C.CF.KLYYv557CCT__[`a
 	r:   c           	     
    t        | | j                        |f|||||d|}	|	j                  }
| j                  |
      }||j                  d   }n|j                  d   }| j
                  j                         j                  |dk7  rt        d      | j
                  j                         j                  d}n||| j
                  j                         j                  k7  j                  |j                  t        j                        }t        j                  |j                  d   |j                  t        j                        }||z  j                  d      }n.d}t        j!                  | j"                  j$                   d       |t        j                  ||j                        |f   }d }|| j'                  |||| j
                  	      }t)        |||	j*                  |	j,                  |	j.                  
      S )Nattention_maskposition_idsr%   inputs_embedsr#   r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.)devicedtypez will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rY   )logitslabelspooled_logitsrR   )lossr\   r%   hidden_states
attentions)getattrrI   last_hidden_staterO   shaperR   rM   pad_token_id
ValueErrortorY   torchint32arangeargmaxr.   r/   r+   r,   loss_functionr   r%   r`   ra   )r3   	input_idsrU   rV   r%   rW   r]   r#   r5   transformer_outputsr`   r\   
batch_sizelast_non_pad_tokennon_pad_masktoken_indicesr^   r_   s                     r9   forwardz(GenericForSequenceClassification.forward   s    8]wtTE[E[7\8
)%+'8
 8
 ,==M* "+J&,,Q/J;;&&(55=*PQ/\]];;&&(55=!#"%)D)D)F)S)SSWWX^XeXeglgrgrsL!LL)<V]]Z_ZeZefM"/,">!F!Fr!J!#>>**+ ,Z Z
 u||Jv}}MOaab%%VFR_hlhshs%tD/ /??-;;*55
 	
r:   NNNNNNN)rm   torch.LongTensor | NonerU   torch.Tensor | NonerV   ru   r%   Cache | NonerW   torch.FloatTensor | Noner]   ru   r#   bool | Noner5   Unpack[TransformersKwargs]returnr   	r,   r;   r<   rI   rF   r   r   rs   r>   r?   s   @r9   rA   rA   q   s      .2.204(,26*.!%8
*8
 ,8
 .	8

 &8
 08
 (8
 8
 -8
 
*8
  8
r:   rA   c                  |     e Zd ZdZ fdZd Zd Zee	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd              Z	 xZ
S )GenericForQuestionAnsweringrB   c                    t         |   |       t        | | j                  t	        j
                  |             t        j                  |j                  d      | _	        | j                          y )N   )r1   rF   rH   rI   r   rJ   rK   rL   rN   
qa_outputsrP   rQ   s     r9   rF   z$GenericForQuestionAnswering.__init__   sQ     d,,i.C.CF.KL))F$6$6: 	r:   c                B    t        | | j                        j                  S Nrb   rI   embed_tokens)r3   s    r9   get_input_embeddingsz0GenericForQuestionAnswering.get_input_embeddings   s    tT334AAAr:   c                :    |t        | | j                        _        y r   r   )r3   values     r9   set_input_embeddingsz0GenericForQuestionAnswering.set_input_embeddings   s    =Bd,,-:r:   c                    t        | | j                        |f||||d|}	|	j                  }
| j                  |
      }|j	                  dd      \  }}|j                  d      j                         }|j                  d      j                         }d }|| | j                  ||||fi |}t        ||||	j                  |	j                        S )N)rU   rV   r%   rW   r   rX   dim)r_   start_logits
end_logitsr`   ra   )rb   rI   rc   r   splitsqueeze
contiguousrl   r   r`   ra   )r3   rm   rU   rV   r%   rW   start_positionsend_positionsr5   outputssequence_outputr\   r   r   r_   s                  r9   rs   z#GenericForQuestionAnswering.forward   s     ,Q749O9O+P,
)%+',
 ,
 "331#)<<r<#: j#++B/::<''+668
&=+D%4%%lJQ^ibhiD+%!!//))
 	
r:   rt   )rm   ru   rU   rv   rV   ru   r%   rw   rW   rx   r   ru   r   ru   r5   rz   r{   r   )r,   r;   r<   rI   rF   r   r   r   r   rs   r>   r?   s   @r9   r~   r~      s    BC  .2.204(,263715%
*%
 ,%
 .	%

 &%
 0%
 1%
 /%
 -%
 
&%
  %
r:   r~   c                  p     e Zd ZdZ fdZee	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd              Z xZS )GenericForTokenClassificationrB   c           	        t         |   |       |j                  | _        t        | | j                  t        j                  |             t        |dd       |j                  }nt        |dd       |j                  }nd}t        j                  |      | _        t        j                  |j                         j                  |j                  t        |dd            | _        | j#                          y )Nclassifier_dropouthidden_dropoutg?token_classification_biasTrD   )r1   rF   rG   rH   rI   r   rJ   rb   r   r   rK   DropoutdropoutrL   rM   rN   rO   rP   )r3   rR   r   r+   s      r9   rF   z&GenericForTokenClassification.__init__   s      ++d,,i.C.CF.KL6/6B!'!:!:V-t4@!'!6!6!$zz"45YY""$00!<dC

 	r:   c           	     ,    t        | | j                        |f|||||d|}	|	j                  }
| j                  |
      }
| j	                  |
      }d }|| j                  ||| j                        }t        |||	j                  |	j                        S )NrT   )r_   r\   r`   ra   )
rb   rI   rc   r   rO   rl   rR   r   r`   ra   )r3   rm   rU   rV   r%   rW   r]   r#   r5   r   r   r\   r_   s                r9   rs   z%GenericForTokenClassification.forward  s     ,Q749O9O+P,
)%+',
 ,
 "33,,7O,%%ffdkkBD$!//))	
 	
r:   rt   )rm   ru   rU   rv   rV   ru   r%   rw   rW   rx   r]   ru   r#   ry   r5   rz   r{   r   r|   r?   s   @r9   r   r      s    *  .2.204(,26*.!%!
*!
 ,!
 .	!

 &!
 0!
 (!
 !
 -!
 
!
  !
r:   r   c                  Z     e Zd Z	 d	 	 	 	 	 	 	 	 	 d fdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ xZS )MtpLayerc                   t         |           || _        || _         ||j                  |j
                        | _         ||j                  |j
                        | _        t        j                  |j                  dz  |j                  d      | _
         |||      | _        |r$ ||j                  |j
                        | _        y d | _        y )Nepsr   FrD   )r1   rF   rR   use_post_normrN   rms_norm_epsenormhnormrK   rL   eh_proj	mtp_block	post_norm)r3   rR   decoder_layer_clsnorm_cls	layer_idxr   r+   s         r9   rF   zMtpLayer.__init__9  s     	*f00f6I6IJ
f00f6I6IJ
yy!3!3a!79K9KRWX*69=R_&"4"4&:M:MNeir:   c                   t        | j                  dd      r8t        j                  | j	                  |      | j                  |      gd      }n7t        j                  | j                  |      | j	                  |      gd      }| j                  |      }	 | j                  |	f||||d|}	| j                  r| j                  |	      }	|	S )Nmtp_hidden_states_firstFrX   r   )rU   position_embeddingsrV   r%   )
rb   rR   rh   catr   r   r   r   r   r   )
r3   rW   previous_hidden_stater   rU   rV   r%   r5   projection_inputr`   s
             r9   rs   zMtpLayer.forwardJ  s     4;; 95A$yy$**5J*KTZZXeMf)gmop$yy$**]*CTZZPeEf)gmop%56&
) 3%+
 
  NN=9Mr:   )T)
rR   r   r   type[nn.Module]r   r   r   intr   bool)rW   torch.Tensorr   r   r   !tuple[torch.Tensor, torch.Tensor]rU   rv   rV   rv   r%   rw   r{   r   )r,   r;   r<   rF   rs   r>   r?   s   @r9   r   r   8  s     #j j +j "	j
 j j"#  , ?	
 , * & 
+r:   r   c                       e Zd ZdZdZdZddgZddgZd fdZddZ	ddZ
	 	 	 	 	 	 	 	 ddZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	Zeddd
       Zedd       Zedd       Z xZS )MtpModelTzshared_head.head.weightzembed_tokens.weightzshared_head.weightc                   t         |   |j                  j                                d| _        || _        |j                         }t        |j                  d         }t        d |j                  d   j                         D              }d| _        d| _        t        | j                  d      r"d| _        | j                  j                  | _        t        j                   t#        |      D cg c]%  }t%        | j                  |||| j                        ' c}      | _        | j                  r7 || j                  j&                  | j                  j(                        | _        | j-                  |       | j/                          y c c}w )NForCausalLMrX   c              3  @   K   | ]  \  }}d |v rt        |        yw)normN)type).0namemodules      r9   	<genexpr>z$MtpModel.__init__.<locals>.<genexpr>{  s'      
f~ L
s   TFchain_hidden_post_normr   )r1   rF   rR   get_mtp_config	loss_typenum_mtp_layersget_decoderr   layersnextnamed_modulesr   use_shared_post_normhasattrr   rK   
ModuleListranger   rN   r   shared_post_normtie_with_main_modelrP   )r3   
main_modelr   
base_model	layer_clsr   kr+   s          r9   rF   zMtpModel.__init__s  sC   **99;<&,++-
**2./	 
 * 1 1" 5 C C E
 
 "$)!4;; 89!&D(,(J(JD% mmX]^lXmnSTXdkk9h4;M;MNn
 $$$,T[[-D-D$++JbJb$cD! 	  , os   .*F	c                    |j                         | _        |j                  | _        |j	                         }t        |dd      | _        y)z8Tie the embedding/head/rotary layer with the main model.
rotary_embN)r   r   lm_headshared_headr   rb   r   )r3   r   r   s      r9   r   zMtpModel.tie_with_main_model  sC     ';;=%--  ++-
!*lDAr:   c                    t        | j                  dd      }|||z  }| j                  |      }t        | j                  dd      }|||j                  d   k  r	|dd|f   }|S )z[Apply the shared head the same way the main model does (muP scaling, unpadded vocab slice).logits_mup_width_multiplierNunpadded_vocab_sizerX   .)rb   rR   r   rd   )r3   r`   
multiplierr\   r   s        r9   _project_to_logitszMtpModel._project_to_logits  s|    T[[*GN
!)J6M!!-0%dkk3H$O*/BV\\RTEU/UC!5"5!556Fr:   c                   | j                   |d|||d}t        | j                  |   dd      }i }|Q|t        v rIt        |   }t	        |t
              r$|j                         D ]  \  }	}
 |
d
i |||	<    n |d
i |||<   nt        d
i ||d<   t        |      dkD  rt        d      dddd	}|j                         D ci c]  \  }}||   | }}}|S c c}}w )z
        Create the (potentially several) masks required for layer `layer_idx`. This relies on the `layer_type`
        attribute of the mtp layer if any, otherwise simply create a causal mask for full attention.
        N)rR   rW   rU   r%   rV   r   
layer_typefull_attentionr   zLYou should have at most 2 masks, 1 for attention, and 1 for linear attentionrU   	conv_mask)r   sliding_attentionlinear_attention )
rR   rb   r   r   
isinstancedictitemsr   lenrf   )r3   r   rW   	mtp_cacherV   mask_kwargsmtp_layer_typemasksmask_functionactual_patternactual_function%internal_layer_expected_kwarg_mappingr   vs                 r9   create_masks_for_mtp_layerz#MtpModel.create_masks_for_mtp_layer  s    kk*"(("
 !Y!7tL%.<b*bB>RM-.7D7J7J7L K3NO,;,Jk,JE.)K )6(D(Dn%&8&G;&GE"#u:>kll /!1 +1
- JOWA6q91<WW Xs   Cc
                
   |j                   d   }g }g }d}t        | j                        D ]  \  }}| j                  |      j	                  |j
                        }| j                  | j                  ||      nd}| j                  ||||      } |||f|||d||
}| j                  r| j                  |      }|t        dd      nt        dd      }| j                  |dd|ddf         }|gt        j                  j                  |d|fd      d|df   j                         }| | j                   ||f| j"                  j$                  |d	|
z  }|j'                  |       |dddddf   j	                  |j
                  
      }|(|	& ||	|j	                  t(        j*                              }|rHt        j                  j-                  dt(        j*                        }t)        j.                  |d      }nt)        j0                  dd      }|j'                  |       t)        j2                  |ddddf   |gd      }t)        j2                  |ddddf   |j5                  |d      gd      }t)        j2                  |ddddf   |ddddf   dz   gd      }|	t)        j2                  |	|gd      }	 t)        j2                  |d      }t)        j2                  |d      }|||fS )a  
        Sample 1 new token for each mtp layers present in this model. Note that the inputs are assumed to be already sliced and correct
        here, i.e. if the main model just processed inputs corresponding to tokens at positions [N-1, N] in the sequence, then from it
        you draft a new token for position N+1, and the `input_ids`/`position_ids`/`attention_mask` here are assumed to correspond to
        data for tokens at positions [N, N+1], i.e. shifted by 1 from the main model, by the newly drafted token. The `last_hidden_states`
        though will correspond to the same as the main model, i.e. positions [N-1, N] in the sequence length dimension.

        `full_input_ids` correspond to the full sequence of `input_ids`, which is used in case we have any `logits_processor` as some
        processors may require to check the length/value of the full previous sequence of ids.
        r   N)rV   )r   rV   r%   rX   i)r   .)
vocab_sizeshift_labelsr[   )r   rZ   r   )num_samplesT)r   keepdimr   )rd   	enumerater   r   rg   rY   r   r   r   r   slicer   rK   
functionalpadr   rl   rR   r   appendrh   float32softmaxmultinomialrk   r   new_ones)r3   rm   last_hidden_statesrU   rV   r   r]   	do_samplelogits_processorfull_input_idsr5   ro   drafted_logitsdrafted_tokensr_   i	mtp_layerrW   r   r   slice_indicesr\   r   next_token_logitsnext_token_scoresprobsnext_mtp_tokennew_candidate_idscandidate_logitss                                r9   rs   zMtpModel.forward  s+   0 __Q'
%dkk2 6	ULAy --i8;;<N<U<UVMMQ__MhLInr  
 33A}iQ]^E!*"" %8) )" " " ((%)%:%:;M%N" 06~E"dO5tCTM,,-?=RS@S-TUF !!}}00!Qt0LSRSRTWU``b***F/3{{/E/ET`dj 
 !!&) &q"ax 0 3 39;K;K 3 L+0J$4^EVEYEYZ_ZgZgEh$i!--.?Ru}}-]!&!2!25a!H!&.?RQU!V!!.1 		9QU#3^"D"MI"YYq!"u(=~?V?VWacd?e'flnoN 99l1ab5&9<23;ORS;S%TZ\]L )!&NN+KQS!Tm6	Up "IIn!< 99^; "2D88r:   c           
        |j                   j                  }|j                   j                         j                  }|j                  j                         }g }|D ]I  }t        j                  d|      }	|	t        |	j                  d            |k  r9|j                  |       K t        |      dk(  r"t        |j                  j                   d      t        j                  dj!                  d |D                    }
|j                   j                         j"                  }| j%                  |j                   j&                  ddd       }t)        |      5   | ||      }d d d        t+        |d d dd d	      \  }}|}d }|w|d
   j-                         D ci c]  \  }}|
j                  |      || }}}|D cg c]4  }t.        j0                  j3                  |      |j5                         v s3|6 }}i }t7               }|D ]s  }t9        |dd      }|j;                  |       |j=                         D ]?  }|||j=                         v s||
j                  |      ,|j?                  |      ||<   A u tA        |||z         D cg c]  }tC        d| dd||z
   d       }}|jE                  tG        d             |jE                  |jH                         tK        ||tM        |||j                   j&                        d       \  }}|D ]  }|jO                  d d d         |jQ                  |       |jR                  r1tU        d| j                   d| dtW        |jR                               |jY                  |       t[        ||d|t\               |S # 1 sw Y   IxY wc c}}w c c}w c c}w )Nz\.(\d+)r   r   z7 does not seem to register any known MTP layer patterns|c              3  (   K   | ]
  }d | d  yw)()Nr   )r   patterns     r9   r   z+MtpModel.from_pretrained.<locals>.<genexpr>?  s     'ZG1WIQ'Zs   FT)pretrained_model_name_or_pathvariant	gguf_fileuse_safetensors
user_agentis_remote_code
weight_mapptcpu)	frameworkrY   zlayers.r(   z.mtp_block.)source_patternstarget_patterns)
add_legacy)weight_mapping
device_maprZ   )rB   
state_dictload_configtp_planzThe following z weights are missing from z9 (checkpoint keys not matching the conversion mapping?): )rB   r  ignore_mismatched_sizesloading_infor.   )/rR   name_or_pathrM   num_hidden_layers"_keys_to_ignore_on_load_unexpectedcopyresearchr   groupr   r   rf   r+   r,   compilejoinr   get_init_contextrZ   r   r   r   ospathbasenamevaluessetr   addkeys	get_slicer   r
   extendr	   _weight_conversionsr   r   __exit__#_adjust_missing_and_unexpected_keysmissing_keysRuntimeErrorsortedr   r   r.   )clsr   r%  r5   r  r,  mtp_patternsfinal_mtp_patternsr  match_object	mtp_regexr   contexts	mtp_modelcheckpoint_filessharded_metadata	mtp_filesmtp_weight_mapr   r   filemtp_state_dictall_pointerfile_pointerNweight_conversionsr*  _s                               r9   from_pretrainedzMtpModel.from_pretrained.  s   (2(9(9(F(F%&--==?QQ "DDIIK  # 	/G99Z9L'C0B0B10E,FIZ,Z%%g.		/
 !"a'
 4 4 = =>>uvwwJJsxx'ZGY'ZZ[	 $**::<KK''
(9(9(?(?tTX& 	8J7I	8 .L*G  .
** %	'!1,!?!E!E!GA9K[K[\]K^Kj1N  +;p$bgg>N>Nt>TXfXmXmXo>opIp e 	BD$TT%HLOOL)!&&( B".18K8K8M3M"*y/?/?/B/N(4(>(>q(AN1%B	B$ ,.?..PQ	
  ")!A'!N_J_I``k@l
 
 	!!">yUZ"[\!!*"@"@A ?%+1jPZPaPaPgPg 
a  	)AJJtT4(	) 	55lC $$ .HIfHg hKKQR^RkRkKlJmo  	%%j1*G$)%	
 c	8 	8  q&
s*   
N(N53N5?4N;4N;O (N2c                     yNTr   rD  s    r9   _can_set_attn_implementationz%MtpModel._can_set_attn_implementation       r:   c                     yrX  r   rY  s    r9   _can_set_experts_implementationz(MtpModel._can_set_experts_implementation  r[  r:   )r   r   r   r   )r   r   )r`   r   r{   r   )r   r   rW   r   r   r   rV   r   )NFNN)rm   r   r  r   rU   rv   rV   rv   r   zMtpCache | Noner]   ru   r  r   r  zLogitsProcessorList | Noner  rv   r{   r   r   )r   r   r{   r   )r{   r   )r,   r;   r<   _supports_sdpa_supports_flex_attn_supports_flash_attnr-  _keys_to_ignore_on_load_missingrF   r   r   r   rs   classmethodrV  rZ  r]  r>   r?   s   @r9   r   r   h  s%    N*CEZ)[&';=R&S#@B	++-9+FN+^j+h +/7;.2W9W9 )W9 ,	W9
 *W9 #W9 (W9 W9 5W9 ,W9 
+W9r f fP    r:   r   );
__future__r   r5  r/  	functoolsr   typingr   rh   torch.nnrK   safetensorsr   cache_utilsr   conversion_mappingr	   core_model_loadingr
   r   masking_utilsr   r   modeling_outputsr   r   r   r   modeling_utilsr   r   r   models.autor   processing_utilsr   utilsr   r   r   r   r   utils.loading_reportr   r   configuration_utilsr   generation.logits_processr   
get_loggerr,   r.   Moduler!   rA   r~   r   r   r   r   r:   r9   <module>rv     s    # 	 	      !  < T U  a ` " $ a a 7 %5> 
		H	%;1 ;1| G
 G
 G
T 9
 9
 9
x ;
 ;
 ;
|-ryy -`w wr:   