
    ^jC+              	          d Z ddlZddlmZ ddlmc mZ ddej                  dej                  dedej                  fdZ	dej                  d	edej                  fd
Z
dej                  d	edej                  fdZ G d dej                        Z G d dej                        Z	 	 ddej                  dee   dz  ddfdZy)zQuantized layers for Gemma: INT2/4/8 packed-weight Linear and Embedding,
plus SRQ (Static Range Quantization) activation rounding.    Nxscalebitsreturnc                 j   |j                  | j                        }d|dz
  z  dz
  }| dz
  }|dk7  }t        j                  ||t        j                  |            }t        j
                  t        j                  | |z        t        |      t        |            |z  }t        j                  |||       S )a[  Apply Static Range Quantization rounding and clipping (in x's dtype).

    A `scale` of 0 means the layer is uncalibrated, in which case this is a no-op. The guard uses
    `torch.where` rather than `scale.item()` so it stays on-device and `torch.compile`-friendly (an
    `.item()` would force a host-device sync and break `fullgraph=True`).
          r   )todtypetorchwhere	ones_likeclamproundfloat)r   r   r   	max_value	min_value
calibrated
safe_scalex_qs           p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/integrations/gemma_quant.py	apply_srqr      s     HHQWWEdQh!#I
QI!JZ0FGJ
++ekk!j.153CU9EU
VYc
cC;;z3**    packedoriginal_widthc                 V   | j                  t        j                        } | dz  j                  t        j                        dz
  }| dz	  j                  t        j                        dz
  } t        j                  ||gd      j
                  g | j                  dd d }|dd|f   S )a  Unpack int4 values from uint8 storage. Two values per byte.

    Each byte: low nibble = first value, high nibble = second value.
    Values are stored unsigned in [0, 15] and shifted to signed [-8, 7].
    Cast to uint8 first so the right shift is logical, not arithmetic.
             dimN.r
   r   uint8int8stackreshapeshape)r   r   lowhighinterleaveds        r   _unpack_int4r,   '   s     YYu{{#FD=

UZZ
(1
,CaKEJJ'!+D:%++sDkr2::RFLL"<MRrRKsO^O+,,r   c                    | j                  t        j                        } | dz  j                  t        j                        dz
  }| dz	  dz  j                  t        j                        dz
  }| dz	  dz  j                  t        j                        dz
  }| dz	  j                  t        j                        dz
  } t        j                  ||||gd      j
                  g | j                  dd d }|dd|f   S )	zUnpack int2 values from uint8 storage. Four values per byte.

    Bits [1:0]/[3:2]/[5:4]/[7:6] hold values 0..3 each, shifted to signed [-2, 1].
       r   r      r    r!   N.r#   )r   r   v0v1v2v3r+   s          r   _unpack_int2r4   5   s    
 YYu{{#F
4-		EJJ	'!	+BQ;$
	"	"5::	.	2BQ;$
	"	"5::	.	2B
A+		%**	%	)B?%++r2r2.B7??WcrARWTVWKsO^O+,,r   c            	            e Zd ZdZ	 	 ddedededef fdZddej                  dz  d	ej                  fd
Z
dej                  d	ej                  fdZd	efdZ xZS )QuantizedLinearzFLinear layer with INT2/4/8 packed weights and SRQ activation rounding.in_featuresout_featuresbiasnum_bitsc                    t         |   |||       || _        |dk(  r/|dz   dz  }t        j                  ||t        j
                        }nZ|dk(  r/|dz   dz  }t        j                  ||t        j
                        }n&t        j                  ||t        j                        }t        j                  |d      | _	        t        j                  t        j                  |dt        j                              | _        t        j                  t        j                  d	t        j                              | _        t        j                  t        j                  d	t        j                              | _        y )
N)r9   r   r.   r   r   r	   Frequires_gradg        )super__init__r:   r   emptyr$   r%   nn	Parameterweightonesfloat32weight_scaletensorinput_activation_scaleoutput_activation_scale)selfr7   r8   r9   r:   	packed_inweight_storage	__class__s          r   r@   zQuantizedLinear.__init__F   s    	l>  q=$qQ.I"[[yTN]$qQ.I"[[yTN"[[{%**UNll>GLLL!5==)YZ ')ll5<<5==3Y&Z#')||ELLEMM4Z'[$r   Nr   r   c                 V   | j                   dk(  r!t        | j                  | j                        }n<| j                   dk(  r!t	        | j                  | j                        }n| j                  }||| j
                  z  S |j                  |      | j
                  j                  |      z  S )u   Dequantize weights (handles int2/int4/int8 storage). If `dtype` is given,
        the math runs in that dtype; otherwise int×fp32 promotion gives fp32.r   r   )r:   r4   rD   r7   r,   rG   r
   )rK   r   int_weightss      r   _dequantize_weightsz#QuantizedLinear._dequantize_weightsa   s     ==A&t{{D4D4DEK]]a&t{{D4D4DEK++K=!2!222~~e$t'8'8';';E'BBBr   r   c                     t        || j                        }t        j                  || j	                  |j
                        | j                        }t        || j                        S N)r   rI   FlinearrQ   r   r9   rJ   )rK   r   outs      r   forwardzQuantizedLinear.forwardn   sL    a445hhq$22177;TYYGd::;;r   c                 r    d| j                    d| j                   d| j                  d u d| j                   S )Nzin_features=z, out_features=z, bias=, num_bits=)r7   r8   r9   r:   rK   s    r   
extra_reprzQuantizedLinear.extra_reprs   sG    4++,OD<M<M;N OIIT)*+dmm_F	
r   )Fr   rS   )__name__
__module____qualname____doc__intboolr@   r   r   TensorrQ   rW   strr[   __classcell__rN   s   @r   r6   r6   C   s    P \\ \ 	\
 \6Ct); Cu|| C< <%,, <

C 
r   r6   c                       e Zd ZdZ	 	 ddededej                  dedef
 fdZe	dej                  fd	       Zd
ej                  dej                  dej                  fdZdej                  dej                  fdZdefdZ xZS )QuantizedEmbeddinga  Embedding with INT2/4/8 packed table, per-row dequant scale, and architectural embed_scale.

    Does NOT subclass `nn.Embedding` because the packed-int storage isn't a usable
    embedding table on its own: indexing `.embedding_quantized[idx]` returns packed
    bytes, not a row of size `embedding_dim`. Callers expect `embed_tokens.weight[idx, :]`
    to return the *dequantized* row, so we expose `weight` as a property (below)
    that returns the dequantized table on demand.
    num_embeddingsembedding_dimoutput_dtypeembed_scaler:   c                 8   t         |           || _        || _        || _        || _        || _        |dk(  r/|dz   dz  }t        j                  ||t        j                        }nZ|dk(  r/|dz   dz  }t        j                  ||t        j                        }n&t        j                  ||t        j                        }t        j                  |d      | _        t        j                  t        j                  |dt        j                              | _        y )Nr   r.   r   r<   r	   Fr=   )r?   r@   rh   ri   scalar_embed_scaler:   rj   r   rA   r$   r%   rB   rC   embedding_quantizedrE   rF   embedding_scale)	rK   rh   ri   rj   rk   r:   
packed_dimembed_storagerN   s	           r   r@   zQuantizedEmbedding.__init__   s     	,*"- ( q='!+1J!KK
%++VM]'!+1J!KK
%++VM!KKUZZXM#%<<U#S !||EJJ~qPUP]P],^_r   r   c                 N    | j                  | j                  | j                        S )zDequantized embedding table (no architectural `embed_scale` applied).

        Mirrors `nn.Embedding.weight` so callers can do `weight[idx, :]` and get
        the same unscaled row they'd get from a non-quantized embedding.
        )rQ   rn   ro   rZ   s    r   rD   zQuantizedEmbedding.weight   s#     ''(@(@$BVBVWWr   
quant_rows
scale_rowsc                 j   | j                   dk(  rt        || j                        }n(| j                   dk(  rt        || j                        }n|}| j                  |j                  d   z  }|j                  |d      }|j                  | j                        |j                  | j                        z  S )zFUnpack int2/int4/int8 + apply per-row block-wise dequantization scale.r   r   r    r!   )r:   r,   ri   r4   r(   repeat_interleaver
   rj   )rK   rs   rt   int_rows
block_sizer   s         r   rQ   z&QuantizedEmbedding._dequantize_weights   s    ==A#J0B0BCH]]a#J0B0BCH!H'':+;+;B+??
,,ZR,@{{4,,-9J9J0KKKr   	input_idsc                     | j                  | j                  |   | j                  |         }|| j                  z  j	                  | j
                        S rS   )rQ   rn   ro   rm   r
   rj   )rK   ry   results      r   rW   zQuantizedEmbedding.forward   sM    ))$*B*B9*MtOcOcdmOno00044T5F5FGGr   c                 n    d| j                    d| j                   d| j                   d| j                   S )Nznum_embeddings=z, embedding_dim=rY   z, embed_scale=)rh   ri   r:   rm   rZ   s    r   r[   zQuantizedEmbedding.extra_repr   sE    d1122B4CUCUBV WnT5L5L4MO	
r   )      ?r   )r\   r]   r^   r_   r`   r   r   r   r@   propertyrb   rD   rQ   
LongTensorrW   rc   r[   rd   re   s   @r   rg   rg   z   s     !`` ` kk	`
 ` `6 X X XLell L LY^YeYe LH!1!1 Hell H
C 
r   rg   modelmodules_to_not_convertc                    ddl }ddlm} |j                  }|j                  }|j
                  xs i }t        |j                               D 	ci c]  \  }}	d| |	 c}	}|r0|j                  dj                  d t        |      D                    nd}
t        | j                               D ]/  \  }} |||      sd|i}|
I|
j                  |      x}6t        fd	|j                         j                         D              }	d|i|	}t!        |t"        j$                        rG|s}t'        d|j(                  |j*                  t-        |d
d      |j.                  j0                  d|}nLt!        |t"        j2                        r0t5        d|j6                  |j8                  |j:                  dud|}n|j=                  d       | j?                  ||       2 | S c c}	}w )aS  Replace `nn.Linear` / `nn.Embedding` modules with `QuantizedLinear` / `QuantizedEmbedding`.

    Per-module bit widths come from `quantization_config.module_quant_configs`.
    `nn.Embedding` modules are only replaced when `quantize_embeddings` is True.
    Modules whose name matches an entry in `modules_to_not_convert` are skipped.
    r   Nr   )should_convert_moduleg|c              3   4   K   | ]  \  }}d | d| d  yw)z(?P<g>)N ).0ipatterns      r   	<genexpr>z,replace_with_quant_layers.<locals>.<genexpr>   s#     ezq'eA3ay2es   r:   c              3   4   K   | ]  \  }}|	|     y wrS   r   )r   r   voverrides_by_groups      r   r   z,replace_with_quant_layers.<locals>.<genexpr>   s!     ida[\[h.q1is   

rm   r}   )rh   ri   rk   rj   )r7   r8   r9   Fr   ) requantizers.quantizers_utilsr   quantize_embeddingsr:   module_quant_configs	enumeratevaluescompilejoinlistnamed_modulessearchnext	groupdictitems
isinstancerB   	Embeddingrg   rh   ri   getattrrD   r   Linearr6   r7   r8   r9   requires_grad_set_submodule)r   quantization_configr   r   r   r   r:   r   r   overridematchernamemoduleoptsmatch
new_moduler   s                   @r   replace_with_quant_layersr      s    C-AA"++H.CCIr
 @II]IdIdIf?gh8AaS'8+h   	

388eYOcEdeef  U0023 .f$T+ABH%W^^D-A$AE#Nieoo>O>U>U>WiiH5H5Dfbll+&+ %44$22#F,@#F#]]00	
 J 		*( "..#00[[, 	J !!%(D*-7.8 LG is   G&)r   )NN)r_   r   torch.nnrB   torch.nn.functional
functionalrT   rb   r`   r   r,   r4   r   r6   Modulerg   r   rc   r   r   r   r   <module>r      s   =    + +ell +# +ell + - -s -u|| -- -s -u|| -4
bii 4
nC
 C
P /39999 !I,9 
	9r   