
    ^j                        d Z ddlZddlZddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlmZ ddlmZmZmZ ddl m!Z!  ejD                  e#      Z$ ed      e G d de                    Z% G d de	jL                        Z' G d de	jL                        Z( G d de	jL                        Z) G d de)      Z* G d de	jL                        Z+e)e*dZ, G d  d!e	jL                        Z- G d" d#e	jL                        Z. G d$ d%e	jL                        Z/ G d& d'e	jL                        Z0 G d( d)e      Z1 G d* d+e	jL                        Z2 G d, d-e	jL                        Z3e G d. d/e             Z4e G d0 d1e4             Z5 G d2 d3e	jL                        Z6 ed4       G d5 d6e4             Z7 G d7 d8e	jL                        Z8 G d9 d:e	jL                        Z9 G d; d<e	jL                        Z: G d= d>e	jL                        Z; G d? d@e	jL                        Z<e G dA dBe4             Z=g dCZ>y)DzPyTorch Data2VecVision model.    N)	dataclass)Optional)nn)CrossEntropyLoss   )initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputSemanticSegmenterOutput)PreTrainedModel)#compile_compatible_method_lru_cache)auto_docstringlogging	torch_int   )Data2VecVisionConfigz7
    Class for outputs of [`Data2VecVisionModel`].
    )custom_introc                       e Zd ZdZy)$Data2VecVisionModelOutputWithPoolingaF  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
        will be returned.
    N)__name__
__module____qualname____doc__     /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.pyr   r   +   s    r   r   c                        e Zd ZdZdeddf fdZdej                  dededej                  fd	Z		 dd
ej                  dej                  dz  dej                  fdZ xZS )Data2VecVisionEmbeddingszc
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.

    configreturnNc                 2   t         |           t        j                  t	        j
                  dd|j                              | _        |j                  r:t        j                  t	        j
                  dd|j                              | _	        nd | _	        t        |      | _        |j                  | _        t        |j                  t        j                   j"                        r|j                  n|j                  |j                  f| _        | j                  j$                  }|j&                  r=t        j                  t	        j
                  d|dz   |j                              | _        nd | _        t        j*                  |j,                        | _        y )Nr   )super__init__r   	Parametertorchzeroshidden_size	cls_tokenuse_mask_token
mask_tokenData2VecVisionPatchEmbeddingspatch_embeddings
patch_size
isinstance
image_sizecollectionsabcIterablenum_patches use_absolute_position_embeddingsposition_embeddingsDropouthidden_dropout_probdropout)selfr"   r6   	__class__s      r   r&   z!Data2VecVisionEmbeddings.__init__B   s$   ekk!Q8J8J&KL   ll5;;q!V=O=O+PQDO"DO =f E ++ &++[__-E-EF ##V%6%67 	
 ++7722')||EKK;QR?TZTfTf4g'hD$'+D$zz&"<"<=r   
embeddingsheightwidthc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Ng      ?r   r      bicubicFsizemodealign_cornersdim)shaper8   r(   jit
is_tracingr0   r   reshapepermuter   
functionalinterpolateviewcat)r<   r>   r?   r@   r6   num_positionsclass_pos_embedpatch_pos_embedrJ   
new_height	new_widthsqrt_num_positionss               r   interpolate_pos_encodingz1Data2VecVisionEmbeddings.interpolate_pos_encodingY   s`    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr   pixel_valuesbool_masked_posc                    |j                   \  }}}}| j                  |      \  }\  }}|j                         \  }	}
}|K| j                  j	                  |	|
d      }|j                  d      j                  |      }|d|z
  z  ||z  z   }| j                  j	                  |	dd      }t        j                  ||fd      }| j                  || j                  |||      z   }| j                  |      }|||ffS NrB   r   rI   )rK   r/   rF   r-   expand	unsqueezetype_asr+   r(   rS   r8   rZ   r;   )r<   r[   r\   _r?   r@   r>   patch_heightpatch_width
batch_sizeseq_lenmask_tokensw
cls_tokenss                 r   forwardz Data2VecVisionEmbeddings.forward   s   
 +001fe262G2G2U/
/\;!+!2
GQ&//00WbIK))"-55kBA#q1u-a?J^^**:r2>
YY
J7Q?
##/#d&C&CJPVX]&^^J\\*-
L+666r   N)r   r   r   r   r   r&   r(   TensorintrZ   
BoolTensorrj   __classcell__r=   s   @r   r!   r!   <   s    
>3 > >.&D5<< &D &DUX &D]b]i]i &DV 487ll7 ))D07 
	7r   r!   c                   Z     e Zd ZdZ fdZdej                  dej                  fdZ xZS )r.   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|d   |d   z  |d   |d   z  f}|| _        || _        || _        || _
        || _        t        j                  ||||      | _        y )Nr   r   kernel_sizestride)r%   r&   r2   r0   num_channelsr*   r1   r3   r4   r5   r6   patch_shaper   Conv2d
projection)	r<   r"   r2   r0   rv   r*   r6   rw   r=   s	           r   r&   z&Data2VecVisionPatchEmbeddings.__init__   s   !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY!!}
15z!}
ST7UV$$(&&))L+:^hir   r[   r#   c                 ^   |j                   \  }}}}|| j                  k7  rt        d      | j                  |j	                  | j                  j
                  j                              }|j                   d   |j                   d   }}|j                  d      j                  dd      }|||ffS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rC   r   r   )	rK   rv   
ValueErrorry   toweightdtypeflatten	transpose)	r<   r[   re   rv   r?   r@   r>   rc   rd   s	            r   rj   z%Data2VecVisionPatchEmbeddings.forward   s    2>2D2D/
L&%4,,,w  __\__T__5K5K5Q5Q%RS
$.$4$4Q$79I9I!9Lk''*44Q:
L+666r   )	r   r   r   r   r&   r(   rl   rj   ro   rp   s   @r   r.   r.      s)    j"7ELL 7U\\ 7r   r.   c                        e Zd Zddededz  ddf fdZ	 	 	 	 ddej                  dedej                  dz  d	ed
ee	   dz  deej                     eej                  ej                  f   z  fdZ
 xZS )Data2VecVisionSelfAttentionNr"   window_sizer#   c                 <   t         |           || _        |j                  |j                  z  dk7  r2t        |d      s&t        d|j                   d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        t        j                  |j                  | j                        | _        t        j                  |j                  | j                  d      | _        t        j                  |j                  | j                        | _        t        j                  |j                         | _        t%        |      | _        | j&                  rt)        ||      | _        y y )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .F)biasr   )r%   r&   r"   r*   num_attention_headshasattrr{   rm   attention_head_sizeall_head_sizer   Linearquerykeyvaluer9   attention_probs_dropout_probr;   boolhas_relative_position_bias"Data2VecVisionRelativePositionBiasrelative_position_biasr<   r"   r   r=   s      r   r&   z$Data2VecVisionSelfAttention.__init__   sP    : ::a?PVXhHi"6#5#5"6 7334A7 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1C%PYYv1143E3EF
zz&"E"EF*.{*;'***LVal*mD' +r   hidden_statesoutput_attentionsr   rZ   
resolutionc                     |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
t        j                  ||	j	                  dd            }|t        j                  | j                        z  }| j                  r[|\  }}|| j                  j                  z  || j                  j                  z  f}|| j                  |||j                   d         z   }|||z   }t        j                   j#                  |d      }| j%                  |      }t        j                  ||
      }|j'                  dddd      j)                         }|j+                         d d | j,                  fz   } |j                  | }|r||f}|S |f}|S )	NrB   r   rC   dim_sizerI   r   r   )rK   r   r   rR   r   r   r   r(   matmulmathsqrtr   r"   r0   r   r   rP   softmaxr;   rO   
contiguousrF   r   )r<   r   r   r   rZ   r   input_shapehidden_shapequery_layer	key_layervalue_layerattention_scoresr?   r@   r   attention_probscontext_layernew_context_layer_shapeoutputss                      r   rj   z#Data2VecVisionSelfAttention.forward   s    $))#2.CCbC$*B*BCjj/44\BLLQPQRHH]+00>HHAN	jj/44\BLLQPQR !<<Y5H5HR5PQ+dii8P8P.QQ **&MFE!T[[%;%;;UdkkF\F\=\]K/$2M2M5@S@STU@V 3N 3  
 "-/2HH --//0@b/I ,,7_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CD6G=/2 O\M]r   rk   FNFN)r   r   r   r   tupler&   r(   rl   r   rm   rj   ro   rp   s   @r   r   r      s    n3 n%$, nZ^ n4 #(6:).(,.||.  . !&t 3	.
 #'. #J%. 
u||	uU\\5<<%?@	@.r   r   c                       e Zd Z	 	 	 	 d	dej                  dedej                  dz  dedee   dz  deej                     eej                  ej                  f   z  fdZy)
Data2VecVisionSdpaSelfAttentionNr   r   r   rZ   r   r#   c           	      :   |r,t         j                  | j                  j                   d       |j                  d d }g |d| j
                  }| j                  |      j                  |      j                  dd      }| j                  |      j                  |      j                  dd      }	| j                  |      j                  |      j                  dd      }
d }| j                  rX|\  }}|| j                  j                  z  || j                  j                  z  f}| j                  |||j                  d         }|
||}n||z  }dt        j                   | j
                        z  }t"        j$                  j&                  j)                  ||	|
|| j*                  r| j                  j,                  ndd|      }|j/                  d	ddd
      j1                         }|j3                         d d | j4                  fz   } |j                  | }|d fS )Nz does not support `output_attentions=True`. The returned attention weights will be `None`. If you want to get attention weights, please set `attn_implementation='eager'` when loading the model.rB   r   rC   r           F)	attn_mask	dropout_p	is_causalscaler   r   r   )loggerwarning_oncer=   r   rK   r   r   rR   r   r   r   r   r"   r0   r   r   r   r(   r   rP   scaled_dot_product_attentiontrainingr   rO   r   rF   r   )r<   r   r   r   rZ   r   r   r   r   r   r   	attn_biasr?   r@   r   scalingr   r   s                     r   rj   z'Data2VecVisionSdpaSelfAttention.forward  s    >>**+ ,D D $))#2.CCbC$*B*BCjj/44\BLLQPQRHH]+00>HHAN	jj/44\BLLQPQR	**&MFE!T[[%;%;;UdkkF\F\=\]K335@S@STU@V 4 I
 "- 2	33	dii 8 899++HHBF--dkk>>UX I 
 &--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CDd""r   r   )	r   r   r   r(   rl   r   r   rm   rj   r   r   r   r   r     s     #(6:).(,/#||/#  /# !&t 3	/#
 #'/# #J%/# 
u||	uU\\5<<%?@	@/#r   r   c                   ~     e Zd ZdZdeddf fdZd	dej                  dej                  dej                  fdZ xZ	S )
Data2VecVisionSelfOutputz
    The residual connection is defined in Data2VecVisionLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r"   r#   Nc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y rk   )	r%   r&   r   r   r*   denser9   r:   r;   r<   r"   r=   s     r   r&   z!Data2VecVisionSelfOutput.__init__G  sB    YYv1163E3EF
zz&"<"<=r   r   input_tensorc                 J    | j                  |      }| j                  |      }|S rk   r   r;   )r<   r   r   gammas       r   rj   z Data2VecVisionSelfOutput.forwardL  $    

=1]3r   rk   )
r   r   r   r   r   r&   r(   rl   rj   ro   rp   s   @r   r   r   A  sE    
>3 > >
U\\  ^c^j^j r   r   )eagersdpac                        e Zd Zddededz  ddf fdZ	 	 	 	 ddej                  dede	d	   d
edee
   dz  deej                     eej                  ej                  f   z  fdZ xZS )Data2VecVisionAttentionNr"   r   r#   c                     t         |           t        |j                     ||      | _        t        |      | _        y )Nr   )r%   r&   &DATA2VEC_VISION_SELF_ATTENTION_CLASSES_attn_implementation	attentionr   outputr   s      r   r&   z Data2VecVisionAttention.__init__[  s8    ?@[@[\
 /v6r   r   r   r   r   rZ   r   c                 l    | j                  |||||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r   )	r<   r   r   r   rZ   r   self_outputsattention_outputr   s	            r   rj   zData2VecVisionAttention.forwardb  sQ     ~~,.DF^`j
  ;;|AF#%QR(88r   rk   r   )r   r   r   r   r   r&   r(   rl   r   r   rm   rj   ro   rp   s   @r   r   r   Z  s    73 7%$, 7Z^ 7 #(QU).(,||   !))M N	
 #' #J% 
u||	uU\\5<<%?@	@r   r   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )Data2VecVisionIntermediater"   r#   Nc                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y rk   )r%   r&   r   r   r*   intermediate_sizer   r1   
hidden_actstrr	   intermediate_act_fnr   s     r   r&   z#Data2VecVisionIntermediate.__init__v  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r   r   c                 J    | j                  |      }| j                  |      }|S rk   )r   r   r<   r   s     r   rj   z"Data2VecVisionIntermediate.forward~  s&    

=100?r   	r   r   r   r   r&   r(   rl   rj   ro   rp   s   @r   r   r   u  s2    93 9 9U\\ ell r   r   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )Data2VecVisionOutputr"   r#   Nc                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y rk   )
r%   r&   r   r   r   r*   r   r9   r:   r;   r   s     r   r&   zData2VecVisionOutput.__init__  sB    YYv779K9KL
zz&"<"<=r   r   c                 J    | j                  |      }| j                  |      }|S rk   r   r   s     r   rj   zData2VecVisionOutput.forward  r   r   r   rp   s   @r   r   r     s2    >3 > >
U\\ ell r   r   c                   r     e Zd ZdZd	deddf fdZdej                  dej                  fdZde	fdZ
 xZS )
Data2VecVisionDropPathzStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    	drop_probr#   Nc                 0    t         |           || _        y rk   )r%   r&   r   )r<   r   r=   s     r   r&   zData2VecVisionDropPath.__init__  s    "r   r   c                 P   | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                        }t	        j                  ||z         }|j                  |      |z  S )Nr   r   r   )r   )r~   device)
r   r   rK   ndimr(   randr~   r   floordiv)r<   r   	keep_probrK   random_tensors        r   rj   zData2VecVisionDropPath.forward  s    >>S   &	$$Q')DM4F4F4J,KK

50C0CML`L`aMI$=>  +m;;r   c                      d| j                    S )Nzp=)r   r<   s    r   
extra_reprz!Data2VecVisionDropPath.extra_repr  s    DNN#$$r   )r   )r   r   r   r   floatr&   r(   rl   rj   r   r   ro   rp   s   @r   r   r     sB    #% #$ #<U\\ <ell <%C %r   r   c                        e Zd ZdZ	 ddededz  deddf fdZ	 	 	 	 ddej                  d	e
d
ej                  dz  de
deeef   dz  deej                     eej                  ej                  f   z  fdZ xZS )Data2VecVisionLayerz?This corresponds to the Block class in the timm implementation.Nr"   r   drop_path_rater#   c                    t         |           |j                  | _        d| _        t	        ||      | _        t        |      | _        t        |      | _	        t        j                  |j                  |j                        | _        |dkD  rt        |      nt        j                          | _        t        j                  |j                  |j                        | _        |j&                  }|dkD  ryt        j(                  |t+        j,                  |j                        z  d      | _        t        j(                  |t+        j,                  |j                        z  d      | _        y d\  | _        | _        y )	Nr   r   epsr   r   T)requires_grad)NN)r%   r&   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r   	LayerNormr*   layer_norm_epslayernorm_beforer   Identity	drop_pathlayernorm_afterlayer_scale_init_valuer'   r(   oneslambda_1lambda_2)r<   r"   r   r   init_valuesr=   s        r   r&   zData2VecVisionLayer.__init__  s    	'-'E'E$0[Q6v>*62 "V-?-?VEZEZ [CQTWCW/?]_]h]h]j!||F,>,>FDYDYZ33?LLuzz&BTBT7U)UeijDMLLuzz&BTBT7U)UeijDM+5(DM4=r   r   r   r   rZ   r   c                    | j                  | j                  |      ||||      }|d   }|dd  }| j                  | j                  |z  }| j                  |      |z   }| j	                  |      }	| j                  |	      }	| j                  |	      }	| j                  | j                  |	z  }	| j                  |	      |z   }	|	f|z   }|S )Nr   r   rZ   r   r   r   )r   r   r  r   r   r   r   r  )
r<   r   r   r   rZ   r   self_attention_outputsr   r   layer_outputs
             r   rj   zData2VecVisionLayer.forward  s     "&!!-0/#9%=! "0 "
 2!4(, ==$#}}/?? '78=H ++M:((6{{<0==$==<7L ~~l3mC/G+r   )Nr   r   )r   r   r   r   r   r   r   r&   r(   rl   r   rm   rj   ro   rp   s   @r   r   r     s    I gj6*69>6^c6	6. #(6:).-1'||'  ' !&t 3	'
 #'' #s(Od*' 
u||	uU\\5<<%?@	@'r   r   c                        e Zd Zdededdf fdZe ed      deeef   de	j                  fd              Zdd	ede	j                  fd
Z xZS )r   r"   r   r#   Nc                     t         |           || _        d|d   z  dz
  d|d   z  dz
  z  dz   | _        t	        j
                  t        j                  | j                  |j                              | _	        y )NrC   r   r   r   )
r%   r&   r   num_relative_distancer   r'   r(   r)   r   relative_position_bias_tabler   s      r   r&   z+Data2VecVisionRelativePositionBias.__init__  sr    &&'+a.&81&<[QR^ASVWAW%X[\%\",.LLKK22F4N4NO-
)r   
   )maxsizec                    d| d   z  dz
  d| d   z  dz
  z  dz   }| d   | d   z  }t        j                  t        j                  | d         t        j                  | d         d      }t        j                  |      }t        j                  |d      }|dddddf   |dddddf   z
  }|j                  ddd      j                         }|dddddfxx   | d   dz
  z  cc<   |dddddfxx   | d   dz
  z  cc<   |dddddfxx   d| d   z  dz
  z  cc<   t        j                  |dz   fdz  |j                        }|j                  d	      |ddddf<   |dz
  |dddf<   |dz
  |dddf<   |dz
  |d
<   |S )z
        This method creates the relative position index, modified to support arbitrary window sizes,
        as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
        rC   r   r   r   ij)indexingN)rF   r~   rB   )r   r   )
r(   meshgridarangestackr   rO   r   r)   r~   sum)r   r  window_areagridcoordscoords_flattenrelative_coordsrelative_position_indexs           r    generate_relative_position_indexzCData2VecVisionRelativePositionBias.generate_relative_position_index  s    "#[^!3a!7AA<NQR<R SVW W "!n{1~5~~ell;q>:ELLUV<XcghT"vq1(At4~aqj7QQ)11!Q:EEG1a KNQ$66 1a KNQ$66 1a AA$6$:: "'++K!O3E3IQ`QfQf"g*9*=*=b*AAB')>)B12&)>)BA&(=(A%&&r   rZ   c                    d| j                   d   z  dz
  }d| j                   d   z  dz
  }d|d   z  dz
  }d|d   z  dz
  }| j                  }| j                  }	||z  dz   }
|d|	dz
   }|j                  d||d      j	                  dddd      }t
        j                  j                  |t        |      t        |      fd      }|j	                  dddd      j                  |
dz
  d      }t        j                  |||	dz
  d g      }| j                  |      }||j                  d         }|j                  |d   |d   z  dz   |d   |d   z  dz   d      }|j	                  ddd      j                         }|rCt
        j                  j                  |j                  d      ||fdd	
      j                  d      }|j                  d      S )zu
        Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
        rC   r   r   r   NrB   bilinear)rF   rG   FrE   )r   r  r  rN   rO   r   rP   rQ   r   r(   rS   r  rR   r   r`   squeeze)r<   r   rZ   r   
old_height	old_widthrW   rX    old_relative_position_bias_tableold_num_relative_distancenew_num_relative_distanceold_sub_tablenew_sub_table new_relative_position_bias_tabler  r   s                   r   rj   z*Data2VecVisionRelativePositionBias.forward  s-    ))!,,q0
((++a/	Q'!+
A&*	+/+L+L($($>$>!$.$:Q$>!89X;TWX;XY%--aJKSSTUWXZ[]^_11:!6	)8L MT^ 2 
 &--aAq9AAB[^_B_acd+099<=VYZ=Z=\]^,
( #'"G"G"T!ABYB^B^_aBb!c "8!<!<N[^+a/Q+a.1PST1TVX"
 "8!?!?1a!H!S!S!U#%']]%>%>&003)#	 &? &
 gaj # &//22r   )FN)r   r   r   r   r   r&   staticmethodr   rm   r(   rl   r  r   rj   ro   rp   s   @r   r   r     ss    
3 
% 
D 
 (4'eCHo '%,, ' 5 '0-3T -3]b]i]i -3r   r   c                        e Zd Zddededz  ddf fdZ	 	 	 	 	 ddej                  deded	ed
ee	e	f   dz  dedee
z  fdZ xZS )Data2VecVisionEncoderNr"   r   r#   c                    t         |           || _        |j                  | _        | j                  rt        ||      | _        t        j                  d|j                  |j                  d      D cg c]  }|j                          }}t        j                  t        |j                        D cg c]!  }t        ||j                   r|nd ||         # c}      | _        d| _        y c c}w c c}w )Nr   r   cpu)r   )r   r   F)r%   r&   r"   !use_shared_relative_position_biasr   r   r   r(   linspacer   num_hidden_layersitemr   
ModuleListranger   use_relative_position_biaslayergradient_checkpointing)r<   r"   r   xdprir=   s         r   r&   zData2VecVisionEncoder.__init__E  s    *0*R*R'***LVal*mD' "'63H3H&JbJbkp!qrAqvvxrr]] v778  $/5/P/PVZ#&q6	

 ',# ss   5C.4&C3r   r   output_hidden_statesrZ   r   return_dictc                    |rdnd }|rdnd }t        | j                        D ]  \  }	}
|r||fz   }| j                  rY|\  }}|| j                  j                  z  || j                  j                  z  f}| j                  |||j                  d         }nd } |
|||||      }|d   }|s||d   fz   } |r||fz   }|st        d |||fD              S t        |||      S )Nr   r   )rZ   r   r  r   c              3   &   K   | ]	  }||  y wrk   r   ).0vs     r   	<genexpr>z0Data2VecVisionEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   )last_hidden_stater   
attentions)		enumerater5  r   r"   r0   r   rK   r   r   )r<   r   r   r:  rZ   r   r;  all_hidden_statesall_self_attentionsr9  layer_moduler?   r@   r   r   layer_outputss                   r   rj   zData2VecVisionEncoder.forwardZ  s6    #7BD$5b4(4 	POA|#$58H$H!.. *%)?)??$++J`J`A`a)-)D)D:R]j]p]pqr]s *E *& *.&("3'=)A%M *!,M &9]1=M<O&O#1	P4   1]4D Dm]4EGZ$[mmm++*
 	
r   rk   )FFFNT)r   r   r   r   r   r&   r(   rl   r   rm   r   rj   ro   rp   s   @r   r+  r+  D  s    ,3 ,%$, ,Z^ ,0 #(%*).-1 /
||/
  /
 #	/

 #'/
 #s(Od*/
 /
 
	 /
r   r+  c                   r     e Zd ZU eed<   dZdZdZdZdgZ	dgZ
dZ ej                          fd       Z xZS )	Data2VecVisionPreTrainedModelr"   data2vec_vision)imager[   Tr   z.*relative_position_index.*c                    t         |   |       t        |t              rwt	        j
                  |j                         |j                  t	        j
                  |j                         |j                   t	        j
                  |j                         yyt        |t              r t	        j
                  |j                         yt        |t              rv|j                  it	        j                  |j                  | j                  j                         t	        j                  |j                   | j                  j                         yyy)zInitialize the weightsN)r%   _init_weightsr1   r!   initzeros_r+   r-   r8   r   r  r   r  	constant_r"   r  r  )r<   moduler=   s     r   rM  z+Data2VecVisionPreTrainedModel._init_weights  s     	f%f67KK(()  ,F--.))5F667 6 BCKK;;< 34*v0R0RSv0R0RS + 5r   )r   r   r   r   __annotations__base_model_prefixinput_modalitiesmain_input_namesupports_gradient_checkpointing_no_split_modules"_keys_to_ignore_on_load_unexpected_supports_sdpar(   no_gradrM  ro   rp   s   @r   rI  rI    sU     ! )!$O&*#./*H)I&NU]]_T Tr   rI  c                        e Zd Zddededdf fdZd Ze	 	 	 	 	 ddej                  dej                  dz  d	edz  d
edz  dededz  deez  fd       Z xZS )Data2VecVisionModelr"   add_pooling_layerr#   Nc                    t         |   |       || _        t        |      | _        t        || j                  j                  j                        | _        |j                  rt        j                         n*t        j                  |j                  |j                        | _        |rt!        |      nd| _        | j%                          y)zw
        add_pooling_layer (bool, *optional*, defaults to `False`):
            Whether to add a pooling layer
        r   r   N)r%   r&   r"   r!   r>   r+  r/   rw   encoderuse_mean_poolingr   r   r   r*   r   	layernormData2VecVisionPoolerpooler	post_init)r<   r"   r]  r=   s      r   r&   zData2VecVisionModel.__init__  s    
 	 26:,VAaAaAmAmn $44BKKM",,vGYGY_e_t_t:u 	 7H*62T 	r   c                 .    | j                   j                  S rk   )r>   r/   r   s    r   get_input_embeddingsz(Data2VecVisionModel.get_input_embeddings  s    ///r   r[   r\   r   r:  rZ   r;  c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  ||      \  }}	|j
                  dd }
| j                  ||||
||      }|d   }| j                  |      }| j                  | j                  |      nd}|s|||fn|f}||dd z   S t        |||j                  |j                        S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        N)r\   rC   )r   r:  r   r;  rZ   r   r   )rA  pooler_outputr   rB  )r"   r   r:  r;  r>   rK   r_  ra  rc  r   r   rB  )r<   r[   r\   r   r:  rZ   r;  kwargsembedding_outputrb   r   encoder_outputssequence_outputpooled_outputhead_outputss                  r   rj   zData2VecVisionModel.forward  s$    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY"oolOo\!!''+
,,/!5!#%= ' 
 *!,..98<8OO4UY?L?XO];_n^pL/!""5553-')77&11	
 	
r   )F)NNNFN)r   r   r   r   r   r&   rf  r   r(   rl   rn   r   r   rj   ro   rp   s   @r   r\  r\    s    3  Y] &0  48)-,0).#',
ll,
 ))D0,
  $;	,

 #Tk,
 #',
 D[,
 
5	5,
 ,
r   r\  c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )rb  r"   r#   Nc                     t         |           |j                  r1t        j                  |j
                  |j                        | _        y d | _        y )Nr   )r%   r&   r`  r   r   r*   r   ra  r   s     r   r&   zData2VecVisionPooler.__init__  sA    KQKbKbBLL++1F1FG 	hl 	r   r   c                     | j                   ,| j                  |d d dd d d f   j                  d            S |d d df   S )Nr   r   )ra  meanr   s     r   rj   zData2VecVisionPooler.forward  sD    BF..B\t~~mAqr1H5::1=>ubopqstptbuur   r   rp   s   @r   rb  rb    s5    
3 
 
vU\\ vell vr   rb  z
    Data2VecVision Model transformer with an image classification head on top (a linear layer on top of the average of
    the final hidden states of the patch tokens) e.g. for ImageNet.
    c                        e Zd Zdeddf fdZe	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dedz  d	ed
edz  de	e
z  fd       Z xZS )$Data2VecVisionForImageClassificationr"   r#   Nc                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NTr]  r   )r%   r&   
num_labelsr\  rJ  r   r   r*   r   
classifierrd  r   s     r   r&   z-Data2VecVisionForImageClassification.__init__  st      ++26TR OUN_N_bcNc"))F$6$68I8IJikititiv 	r   r[   labelsr   r:  rZ   r;  c                 \   ||n| j                   j                  }| j                  |||||      }|r|j                  n|d   }	| j	                  |	      }
d}|| j                  ||
| j                         }|s|
f|dd z   }||f|z   S |S t        ||
|j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r:  rZ   r;  r   rC   losslogitsr   rB  )	r"   r;  rJ  rh  rx  loss_functionr   r   rB  )r<   r[   ry  r   r:  rZ   r;  ri  r   rm  r~  r}  r   s                r   rj   z,Data2VecVisionForImageClassification.forward  s    " &1%<k$++BYBY&&/!5%=# ' 
 2=--'!*/%%ffdkkBDY,F)-)9TGf$EvE$!//))	
 	
r   NNNNFN)r   r   r   r   r&   r   r(   rl   r   r   r   rj   ro   rp   s   @r   rt  rt     s    
3 
 
  -1&*)-,0).#'*
llT)*
 t#*
  $;	*

 #Tk*
 #'*
 D[*
 
&	&*
 *
r   rt  c                        e Zd Z	 	 	 	 	 	 	 ddededeeeef   z  dedeeeef   z  ez  dedeeeef   z  ded	ef fd
Zdej                  dej                  fdZ
 xZS )Data2VecVisionConvLayerin_channelsout_channelsrt   ru   paddingr   dilationgroups
activationc
           
          t         
|           t        j                  ||||||||      | _        t        j
                  |      | _        |	t        |	   | _	        y t        j                         | _	        y )N)r  r  rt   ru   r  r  r  r   )
r%   r&   r   rx   convolutionBatchNorm2dnormalizationr	   r   r  )r<   r  r  rt   ru   r  r   r  r  r  r=   s             r   r&   z Data2VecVisionConvLayer.__init__D  si     	99#%#	
  ^^L90:0F&,BKKMr   r   r#   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rk   )r  r  r  r   s     r   rj   zData2VecVisionConvLayer.forward^  s6    ((7**=96r   )r   r   r   Fr   r   relu)r   r   r   rm   r   r   r   r&   r(   rl   rj   ro   rp   s   @r   r  r  C  s    
 .//0*+ ZZ Z 5c?*	Z
 Z uS#X&,Z Z c3h'Z Z Z4U\\ ell r   r  c                   v     e Zd Zdedededdf fdZdej                  deeef   dej                  fd	Z xZ	S )
!Data2VecVisionPyramidPoolingBlock
pool_scaler  channelsr#   Nc                 |    t         |           t        j                  |      | _        t        ||d      | _        y )Nr   rt   )r%   r&   r   AdaptiveAvgPool2dpoolingr  conv)r<   r  r  r  r=   s       r   r&   z*Data2VecVisionPyramidPoolingBlock.__init__g  s0    ++J7+KqQ	r   inputrF   c                     | j                  |      }| j                  |      }t        j                  j	                  ||dd      }|S )Nr  FrE   )r  r  r   rP   rQ   )r<   r  rF   hidden_states       r   rj   z)Data2VecVisionPyramidPoolingBlock.forwardl  sB    ||E*yy.}}00Dzin0or   )
r   r   r   rm   r&   r(   rl   r   rj   ro   rp   s   @r   r  r  f  sS    R3 RS RC RD R
U\\ sCx U\\ r   r  c                   |     e Zd ZdZdeedf   dededdf fdZd	ej                  de	ej                     fd
Z
 xZS )"Data2VecVisionPyramidPoolingModuleak  
    Pyramid Pooling Module (PPM) used in PSPNet.

    Args:
        pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
            Module.
        in_channels (int): Input channels.
        channels (int): Channels after modules, before conv_seg.

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    pool_scales.r  r  r#   Nc           
          t         |           || _        || _        || _        t        j                  |D cg c]  }t        |||       c}      | _        y c c}w )N)r  r  r  )	r%   r&   r  r  r  r   r2  r  blocks)r<   r  r  r  r  r=   s        r   r&   z+Data2VecVisionPyramidPoolingModule.__init__  s]    && mm #. 2ZU`kst
s   Ar   c                 v    |j                         dd  }| j                  D cg c]  } |||       c}S c c}w )NrC   )rF   )rF   r  )r<   r   original_sizeblocks       r   rj   z*Data2VecVisionPyramidPoolingModule.forward  s6    %**,QR0FJkkRUm-8RRRs   6)r   r   r   r   r   rm   r&   r(   rl   listrj   ro   rp   s   @r   r  r  t  sV    


E#s(O 

# 

QT 

Y] 

SU\\ Sd5<<6H Sr   r  c                        e Zd ZdZdeddf fdZdeej                     dej                  fdZ	deej                     dej                  fd	Z
 xZS )
Data2VecVisionUperHeadz
    Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
    [UPerNet](https://huggingface.co/papers/1807.10221).

    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r"   r#   Nc           	         t         |           |j                  | _        |j                  gdz  | _        |j                  | _        t        j                  | j
                  |j                  d      | _	        t        | j                  | j                  d   | j
                        | _        t        | j                  d   t        | j                        | j
                  z  z   | j
                  dd      | _        t        j                         | _        t        j                         | _        | j                  d d D ]o  }| j                   j%                  t        || j
                  d             | j"                  j%                  t        | j
                  | j
                  dd             q t        t        | j                        | j
                  z  | j
                  dd      | _        y )N   r   r  rB   r   rt   r  )r%   r&   r  r*   r  r  r   rx   rw  rx  r  psp_modulesr  lenpsp_bottleneckr2  lateral_convs	fpn_convsappendfpn_bottleneck)r<   r"   r  r=   s      r   r&   zData2VecVisionUperHead.__init__  s   !--"../!3**))DMM63D3DRST >R MM

 6R 3t'7'7#84==#HHMM	
  ]]_++CR0 	sK%%&=k4==fg&hiNN!!"9$--deop"qr	s 6  !DMM1MM	
r   r   c                     |d   }t        j                  |g| j                  |      d      }| j                  |      S r^   )r(   rS   r  r  )r<   r   r  s      r   psp_forwardz"Data2VecVisionUperHead.psp_forward  sA    $R(yy,!P1A1A,1O!PVWX""<00r   encoder_hidden_statesc                 6   g }t        | j                  |      D ]  \  }}|j                   ||              |j                  | j                  |             t	        |      }t        |dz
  dd      D ]L  }||dz
     j                  dd  }||dz
     t        j                  j                  ||   |dd      z   ||dz
  <   N g }t        |dz
        D ])  }|j                   | j                  |   ||                + |j                  |d          t        |dz
  dd      D ];  }t        j                  j                  ||   |d   j                  dd  dd      ||<   = t        j                  |d      }| j                  |      }	| j                  |	      }	|	S )	Nr   r   rB   rC   r  FrE   rI   )zipr  r  r  r  r3  rK   r   rP   rQ   r  r(   rS   r  rx  )
r<   r  lateralslateral_convr  used_backbone_levelsr9  
prev_shapefpn_outsr   s
             r   rj   zData2VecVisionUperHead.forward  s   *-d.@.@BW*X 	8&L,OOL67	8 	(()>?@  #8}+a/B7 	A!!a%..qr2J&q1uo0I0I*:U 1J 1 HQUO	 +a/0 	<AOO-DNN1-hqk:;	< 	%+a/B7 	A--33(1+"3"3AB"7jX] 4 HQK	 99X1-$$X.(r   )r   r   r   r   r   r&   r  r(   rl   r  rj   ro   rp   s   @r   r  r    s]     
3  
  
D1ell); 1 1
T%,,-? ELL r   r  c                        e Zd ZdZ	 	 	 ddedededeeeef   z  ddf
 fdZd	ee	j                     de	j                  fd
Z xZS )Data2VecVisionFCNHeada  
    Fully Convolution Networks for Semantic Segmentation. This head is implemented of
    [FCNNet](https://huggingface.co/papers/1411.4038>).

    Args:
        config (Data2VecVisionConfig): Configuration.
        in_channels
        kernel_size (int): The kernel size for convs in the head. Default: 3.
        dilation (int): The dilation rate for convs in the head. Default: 1.


    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
    r"   in_indexrt   r  r#   Nc           
      0   t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _	        || _
        |dz  |z  }t        j                         | _        | j                  dkD  r| j                  j                  t        | j                  | j
                  |||             t!        | j                  dz
        D ]?  }| j                  j                  t        | j
                  | j
                  |||             A | j                  r8t        | j                  | j
                  z   | j
                  ||dz        | _        t        j$                  | j
                  |j&                  d      | _        y )NrC   r   )rt   r  r  r   r  r  )r%   r&   r*   r  auxiliary_channelsr  auxiliary_num_convs	num_convsauxiliary_concat_inputconcat_inputr  r   r2  convsr  r  r3  conv_catrx   rw  rx  )r<   r"   r  rt   r  conv_paddingrb   r=   s          r   r&   zData2VecVisionFCNHead.__init__  sQ    	!--1133"99 #q(H4]]_
>>AJJ'$$dmmVbmu
 4>>A-. 	

!!+$/ ,!)	 3  4==0$--[bmqrbrDM ))DMM63D3DRSTr   r  c                     || j                      }|}| j                  D ]
  } ||      } | j                  r(| j                  t	        j
                  ||gd            }| j                  |      }|S )Nr   rI   )r  r  r  r  r(   rS   rx  )r<   r  residualr   r  s        r   rj   zData2VecVisionFCNHead.forward  sn    (7 JJ 	0D /M	0 MM%))X}4MST*UVM6r   )rC   r   r   )r   r   r   r   r   rm   r   r&   r  r(   rl   rj   ro   rp   s   @r   r  r    s|    " *+%U$%U %U 	%U
 c3h'%U 
%UNT%,,-? ELL r   r  c                        e Zd Zdeddf fdZd Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  de	dz  d	e	dz  d
e	de	dz  de
ez  fd       Z xZS )%Data2VecVisionForSemanticSegmentationr"   r#   Nc                 x   t         |   |       |j                  | _        t        |d      | _        t        | j                  j                        dk7  rt        d      t        j                  t        j                  |j                  |j                  dd      t        j                  |j                        t        j                         t        j                  |j                  |j                  dd            | _        t        j                  t        j                  |j                  |j                  dd            | _        t        j"                         | _        t        j&                  dd      | _        t+        |      | _        |j.                  rt1        |      nd | _        | j5                          y )NFrv  r  zData2VecVisionForSemanticSegmentation requires config.out_indices to be a list of 4 integers, specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of a base-sized architecture.rC   rs   )r%   r&   rw  r\  rJ  r  r"   out_indicesr{   r   
SequentialConvTranspose2dr*   r  GELUfpn1fpn2r   fpn3	MaxPool2dfpn4r  decode_headuse_auxiliary_headr  auxiliary_headrd  r   s     r   r&   z.Data2VecVisionForSemanticSegmentation.__init__)  sQ     ++26US t{{&&'1,- 
 MMv1163E3EST]^_NN6--.GGIv1163E3EST]^_	
	 MMv1163E3EST]^_
	 KKM	LLQq9	 2&9?E?X?X3F;^b 	r   c                 n   t         j                  j                  ||j                  dd  dd      }|0t         j                  j                  ||j                  dd  dd      }t	        | j
                  j                        } |||      }|}|% ||      }	|| j
                  j                  |	z  z  }|S )Nr   r  FrE   )ignore_index)r   rP   rQ   rK   r   r"   semantic_loss_ignore_indexauxiliary_loss_weight)
r<   r~  auxiliary_logitsry  upsampled_logitsupsampled_auxiliary_logitsloss_fct	main_lossr}  auxiliary_losss
             r   compute_lossz2Data2VecVisionForSemanticSegmentation.compute_lossI  s    ==44bc*5 5 
 ')+)B)B v||BC'8zY^ *C *& $1W1WX-v6	'%&@&INDKK55FFDr   r[   ry  r   r:  rZ   r;  c           	      R   ||n| j                   j                  }||n| j                   j                  }|$| j                   j                  dk(  rt	        d      | j                  ||d||      }|r|j                  n|d   }	t        |	      D 
cg c]#  \  }
}|
dz   | j                   j                  v s"|% }}
}|j                  d   }| j                   j                  | j                   j                  z  }|D cg c]3  }|ddddddf   j                  ddd      j                  |d||      5 }}| j                  | j                  | j                   | j"                  g}t%        t'        |            D ]  } ||   ||         ||<    | j)                  |      }d}| j*                  | j+                  |      }d}|| j-                  |||      }|s|r
|f|dd z   }n	|f|dd z   }||f|z   S |S t/        |||r|j                  nd|j0                  	      S c c}}
w c c}w )
a  
        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, Data2VecVisionForSemanticSegmentation
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/data2vec-vision-base")
        >>> model = Data2VecVisionForSemanticSegmentation.from_pretrained("facebook/data2vec-vision-base")

        >>> inputs = image_processor(images=image, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> # logits are of shape (batch_size, num_labels, height, width)
        >>> logits = outputs.logits
        ```Nr   z/The number of labels should be greater than oneTr{  r   rC   rB   r|  )r"   r;  r:  rw  r{   rJ  r   rC  r  rK   r2   r0   rO   rN   r  r  r  r  r3  r  r  r  r  r   rB  )r<   r[   ry  r   r:  rZ   r;  ri  r   r  idxfeaturefeaturesre   patch_resolutionr7  opsr9  r~  r  r}  r   s                         r   rj   z-Data2VecVisionForSemanticSegmentation.forward\  sa   H &1%<k$++BYBY$8$D $++JjJj 	 $++"8"8A"=NOO&&/!%%=# ' 
 :E 5 5'RS* 1::O0PwWTWZ[T[_c_j_j_v_vTvGww!''*
;;11T[[5K5KKnv
ijAaQhK1a(00RAQScd
 

 yy$))TYY		:s8}% 	.A #a&!-HQK	. !!(+*#228<$$V-=vFD# WQR[0 WQR[0)-)9TGf$EvE&3G'//T))	
 	
; x
s   #H6H=8H$r  )r   r   r   r   r&   r  r   r(   rl   r   r   r   rj   ro   rp   s   @r   r  r  &  s    3  @&  -1&*)-,0).#'Y
llT)Y
 t#Y
  $;	Y

 #TkY
 #'Y
 D[Y
 
(	(Y
 Y
r   r  )rt  r  r\  rI  )?r   collections.abcr3   r   dataclassesr   typingr   r(   r   torch.nnr    r   rN  activationsr	   modeling_layersr
   modeling_outputsr   r   r   r   modeling_utilsr   pytorch_utilsr   utilsr   r   r   configuration_data2vec_visionr   
get_loggerr   r   r   Moduler!   r.   r   r   r   r   r   r   r   r   r   r   r+  rI  r\  rb  rt  r  r  r  r  r  r  __all__r   r   r   <module>r     sU   $   !    % & ! 9  . @ 7 7 ? 
		H	% 
 +E  \7ryy \7@#7BII #7NF")) FT0#&A 0#hryy & )+* &bii 6 "
299 
%RYY %2>4 >DQ3 Q3jE
BII E
P TO T T8 D
7 D
 D
P	v299 	v 8
+H 8
8
xbii F
		 
S S>NRYY Nd>BII >B N
,I N
 N
br   