^j< | d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z ddl
m Z dd l m
Z
mZ dd
lmZmZ dd lmZ dd lmZ dd
lmZ ddlmZmZmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z. ej^ e0 Z1dZ2dZ3 ed e G d de Z4 G d de) Z5 G d d e! Z6 G d d ejn Z8 G d d e. Z9 G d! d"e- Z: G d# d$e( Z; G d% d&e;e' Z< G d' d(e# Z= G d) d*e% Z> G d+ d,e& Z? G d- d.e$ Z@g d/ZAy)0zLG AI Research EXAONE Lab )CallableN)strict)nn )Cache DynamicCache)PreTrainedConfig)create_causal_mask!create_sliding_window_causal_mask)BaseModelOutputWithPastCausalLMOutputWithPast)RopeParameters)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargsauto_docstringlogging)merge_with_config_defaults)capture_outputs )Gemma2RotaryEmbedding) LlamaForCausalLMLlamaForQuestionAnswering LlamaForSequenceClassificationLlamaForTokenClassification
LlamaModelLlamaPreTrainedModel LlamaRMSNormapply_rotary_pos_embeager_attention_forward)Olmo2DecoderLayerOlmo2MLPzLGAI-EXAONE/EXAONE-4.0-32B
Exaone4Config)
checkpointc
e Zd ZU dZdZdgZdddddddddd Zdgd gfd
d gd
gfd
gd
gfd Zd
Ze e
d< dZ e e
d< dZ e e
d< dZ
e e
d< dZe e
d< dZe e
d< dZee
d< dZe e
d< dZee
d < d Zee
d < d Zee
d < d!Ze d"z e
d#< d$Ze ee z d"z e
d%< d"Ze d"z e
d&< d'Z ee
d(< d"Z e e z d"z e
d)< d*Z ee z e
d+< dZ!e d"z e
d,< d-Z"ee z d"z e
d.< d"Z#ee d"z e
d/< fd0Z$ xZ%S )1r# a~
sliding_window_pattern (`str`, *optional*):
The pattern to use for sliding window attention. Can be one of:
- `None`: No sliding window attention is used
- `int`: Every `sliding_window` layers, use global attention, else use local attention.
- `str`: A sequence of "L" (local attention) and "G" (global attention) characters that defines the
attention pattern. The pattern starts from layer 0 and repeats every `sliding_window` layers. The
final layer always uses global attention regardless of the pattern.
For instance, sliding_window_pattern="LLLG" same as sliding_window=4, which means:
- Layer 0, 1, 2: local attention,
- Layer 3: global attention,
...(repeated)
Example:
```python
>>> from transformers import Exaone4Model, Exaone4Config
>>> # Initializing a EXAONE configuration
>>> configuration = Exaone4Config()
>>> # Initializing a model from configuration
>>> model = Exaone4Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```exaone4past_key_valuescolwise replicated_with_grad_allreducerowwise) zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj input_ids
inputs_embeds
hidden_statesattention_mask) embed_tokenslayersnormi
vocab_sizei hidden_sizei @ intermediate_size num_hidden_layersnum_attention_headsnum_key_value_headssilu
hidden_acti max_position_embeddingsg{Gz?initializer_rangegh㈵> rms_norm_epsT use_cacher N bos_token_idr eos_token_id pad_token_idFtie_word_embeddingsrope_parameters attention_dropoutsliding_window sliding_window_pattern layer_typesc | j d| _ | j Nt | j D cg c]* }|dz | j z dk7 r|| j k rdnd, c}| _ t | di | y c c}w )Nr sliding_attentionfull_attention )rF rH rI ranger6 super
__post_init__)selfkwargsi __class__s v/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/exaone4/modular_exaone4.pyrQ zExaone4Config.__post_init__ s &*+D ' #
t 5 56 Ut : :;q@QI_I_E_ $%& D '' s /A?)&__name__
__module__ __qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr2 int__annotations__r3 r4 r6 r7 r8 r: strr; r< floatr= r> boolr? r@ listrA rB rC r dictrE rF rH rI rQ
__classcell__rU s @rV r# r# 9 s 8 J#4"5 &/%.%.%E%E%."+ )"+
& (9:#%56 8IJ!"_$56 J K "s" s !! !! J #'S' #u# L% It L#* +,L#S /D(, #L#*# %%48O^d*T18%(us{(!%NC$J%/0C#I ,0$(KcT!( ( ( c e Zd Zy)Exaone4RMSNormNrW rX rY rN rh rV rj rj rh rj c e Zd Zy)Exaone4RotaryEmbeddingNrk rN rh rV rn rn rl rh rn c e Zd Zdedef fd Z d dej deej ej f dej dz de dz d e
e d
eej ej dz eej dz f f d Z xZ
S )
Exaone4Attentionconfig layer_idxc t | || _ || _ |j | _ |j
| _ |j | _ t |d|j |j z | _ |j |j
z | _ |j | _
d| _ | j dz | _ |j | _
|j | _ t |d r|j | nd }|dk( | _ t% j& | j | j | j z d | _ t% j& | j | j
| j z d | _ t% j& | j | j
| j z d | _ t% j& | j | j z | j d | _ t1 | j |j2 | _ t1 | j |j2 | _ y ) Nhead_dimTg rI rL F)biaseps) rP __init__rq rr r7 r8 r3 getattrrt num_key_value_groupsrE is_causalscalingrF rH hasattrrI
is_slidingr Linearq_projk_projv_projo_projrj r= q_normk_norm)rR rq rr
layer_typerU s rV rx zExaone4Attention.__init__ s
"#)#=#= #)#=#= !--
F4F4F&JdJd4de
$*$>$>&B\B\$\!!'!9!9 }}d* $ 3 3&,&C&C#6=fm6TV'' 2Z^
$(;;ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii 8 84== H$JZJZafg $T]]8K8KL $T]]8K8KL rh Nr- position_embeddingsr. r' rS returnc |j d d }g |d| j }| j | j | j dd }| j | j | j dd } | j
| j | j dd }
| j | }| j | } |\ } } | j | j rt || | | \ }} | |j | |
| j \ } }
t j | j j" t$ }
|
| || |
|f| j& sdn | j( | j* | j r | j nd d|\ }} |j, g |d j/ }| j1 | }||fS )NrK r rD )dropoutr| rF )shapert r view transposer r r r rF r~ r updaterr r
get_interfacerq _attn_implementationr trainingrE r| reshape
contiguousr )rR r- r r. r' rS input_shape hidden_shape query_states
key_states value_statescossinattention_interface attn_output attn_weightss rV forwardzExaone4Attention.forward s $))#2. 88b8$--8 {{=166|DNNqRST [[ /44\BLLQPQR
{{=166|DNNqRST {{<0 [[,
&S &$//';L*VY[^'_ $L* &'6'='=j,X\XfXf'g $J (?(M(M KK , ,.E)
%8
%
$}}C$2H2HLL26//4..t
%
%
! \ *k));;;;FFH kk+. L((rh )NN)rW rX rY r# r_ rx torchTensortupler r r r rf rg s @rV rp rp s M} M M: /3(, -) ||-) #5<< #= >-) t+ -)
-) +,
-)
u||U\\D 0% 2E2LL M-)rh rp c e Zd Zy)
Exaone4MLPNrk rN rh rV r r rl rh r c e Zd Zy)Exaone4DecoderLayerNrk rN rh rV r r rl rh r c e Zd ZeZdgZy)Exaone4PreTrainedModelr N)rW rX rY r# config_class_no_split_modulesrN rh rV r r s L./rh r c e Zd Zdef fd Zee d
dej dz dej dz dej dz de
dz dej dz d e dz d
e
e d eez fd Z xZS ) Exaone4Modelrq c $ t | | t j t |j
D cg c] }t
|| c} | _ t |j |j | _ | j y c c}w )Nrv )
rP rx r
ModuleListrO r6 r r0 rj r3 r= r1 post_init)rR rq rr rU s rV rx zExaone4Model.__init__ so
mmEJ6KcKcEd e
3 e
#6#5#56;N;NO
fs B
Nr+ r. position_idsr' r, r> rS r c
|d u |d uz r t d || j | }|r|t | j }|V||j nd}t j |j d |j |z }|j d }t |x} t sE| j ||||d}
dt d i |
i} d| j j v rt d i |
| d< |} | j | | } t! | j" D ]0 \ }
}| j j |
} || f| | |||| d |} 2 | j% | } t' | |r|
S d
S ) Nz:You must specify exactly one of input_ids or inputs_embeds)rq r rK )device)rq r, r. r' r rM rL )r. r r' r> r )last_hidden_stater' rN )
ValueErrorr/ r rq get_seq_lengthr aranger r unsqueeze
isinstancere r
rI r
rotary_emb enumerater0 r1 r )rR r+ r. r r' r, r> rS past_seen_tokenscausal_mask_mapping mask_kwargsr- r rT
decoder_layerr s rV r zExaone4Model.forward s
-t"; < YZ Z - -i 8M 0 *$++ >O CRC^ = = ?de <<
(;(;A(>}G[G[\_ooL'11!4L ?-F ++!."0#2 , K !"4"C{"C# #dkk&=&==;\;k_j;k#$78%
"oom\ J )$++ 6
A} 003J ) 2: > ) /#$7
M
-0
& +/8O
>B
rh )NNNNNN)rW rX rY r# rx r r r
LongTensorr r FloatTensorrc r r r r r rf rg s @rV r r s } .2.204(,26!%=
##d*=
t+=
&&- =
=
((4/
=
$;=
+,=
( (=
=
rh r c e Zd Z d
dej dz dej
dz dej dz dedz dej dz dej dz dedz d e ej
z d
e
e d e f fd
Z
xZS )Exaone4ForCausalLMNr+ r. r r' r, labelsr> logits_to_keeprS r c 6 t
| d||||||||d| y)u
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Example:
```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer
>>> model = AutoModelForCausalLM.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")
>>> tokenizer = AutoTokenizer.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")
>>> prompt = "Explain how wonderful you are"
>>> messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": prompt}
]
>>> input_ids = tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_tensors="pt",
enable_thinking=False,
)
>>> output = model.generate(input_ids, max_new_tokens=128)
>>> tokenizer.decode(output[0], skip_special_tokens=False)
"[|system|]\nYou are a helpful assistant.[|endofturn|]\n[|user|]\nExplain how wonderful you are[|endofturn|]\n[|assistant|]\n\n\n\n\nOh, thank you for such a kind and lovely question! 😊 \n\nI’m *so* wonderful because I’m here to make your life easier, brighter, and more fun! Whether you need help with: \n\n✨ **Learning** – I can explain anything, from quantum physics to baking the perfect cake! \n💡 **Creativity** – Need a poem, story, or a wild idea? I’ve got you covered! \n🤖 **Problem-solving** – Stuck on a math problem or a tricky decision? I’ll help you figure it out"
```
)r+ r. r r' r, r r> r NrN )rP r ) rR r+ r. r r' r, r r> r rS rU s rV r zExaone4ForCausalLM.forward< s9 V
)% +' )
rh )NNNNNNNr )rW rX rY r r r r r rc r_ r r r
r rf rg s @rV r r ; s .2.204(,26*.!%-.5
##d*5
t+5
&&- 5
5
((4/
5
4'5
$;5
ell*5
+,5
5
5
rh r c e Zd Zy) Exaone4ForSequenceClassificationNrk rN rh rV r r t rl rh r c e Zd Zy) Exaone4ForTokenClassificationNrk rN rh rV r r x rl rh r c e Zd Zy)Exaone4ForQuestionAnsweringNrk rN rh rV r r | rl rh r )r# r r r r r r )BrZ collections.abcr r huggingface_hub.dataclassesr r cache_utilsr r configuration_utilsr
masking_utilsr
r modeling_outputsr r
modeling_rope_utilsr modeling_utilsr processing_utilsr utilsr r r
utils.genericr utils.output_capturingr gemma2.modeling_gemma2r llama.modeling_llamar r r r r r r r r olmo2.modeling_olmo2r! r"
get_loggerrW logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr# rj rn Modulerp r r r r r r r r __all__rN rh rV r s? $ . . 3 R 2 5 & @ @ 7 5 :
?
H %2 ! 78Q($ Q( 9Q(h \ 2 G)ryy G)T + 0 1 0
J
): J
Z6
) 6
r 'E $? "; rh