
    ^j(                        d Z ddlmZ ddlmZ ddlmZ ddlZddl	m
Z
 dd	lmZmZ  e
j                  e      Ze G d
 d             Ze G d d             ZddZddZddZddZg dZy)z<Template loading and validation for response_template dicts.    )annotations)	dataclass)AnyN   )logging   )CONTENT_PARSERSvalidate_transform_stringsc                      e Zd ZU ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   y)ResponseTemplateFieldstrnamer   open_relist[str] | Noneopen_literalsboolopen_literal_can_extendclose_reclose_literalsclose_literal_can_extendcontentdictcontent_argsrepeatsoptional	transformtransform_eachN)__name__
__module____qualname____annotations__     }/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/utils/chat_parsing/response_templates.pyr   r      sI    
IL##!!M$$""LMNNr#   r   c                  P    e Zd ZU ded<   ded<   ded<   dZded	<   dZd
ed<   ddZy)ResponseTemplater   defaultsz dict[str, ResponseTemplateField]fieldsr   start_anchor_reNz
str | Noneimplicitr   start_anchor_literalsc                    d}| j                   j                  |      D ]  }|j                         } |+| j                  dnd}t        j                  d| d       |S ||d S )a  When parsing responses, we can't just parse the tokens generated
        by the model. This is because chat templates often include early parts of the message such as <start_of_turn>
        or <think> tokens in the prefill. Therefore, we take the whole chat as input, but truncate to the start
        of the most recent assistant message, which can include tokens from the template as well as the output.Nstart_anchorstart_anchor_patternzresponse_template defines z_ but the anchor was not found in the prefix; the parser will process the entire prefix instead.)r)   finditerendr+   loggerinfo)selftextlast_endmkinds        r$   truncate_past_last_anchorz*ResponseTemplate.truncate_past_last_anchor7   s    
  $%%..t4 	AuuwH	%)%?%?%K>QgDKK,TF 3T T KHIr#   )r4   r   returnr   )r   r   r    r!   r*   r+   r8   r"   r#   r$   r&   r&   /   s.    N,,Hj.2+2r#   r&   c           	     \  	 ||v r||v rt        |  d| d| d      ||v r,||   }t        |t              r|g	nt        |t              rU|st        |  d| d      t	        d |D              st        |  d| d      t        t
        j                  |            	n&t        |  d| dt        |      j                         t        d	 	D              rt        |  d| d
      t        	t        d      }t        	fd	D              }dj                  d |D              }t        j                  |t        j                        	|fS ||v r+	 t        j                  ||   t        j                        ddfS y# t        j                   $ r}t        |  d| d|       |d}~ww xY w)a  Compile a region's start/end anchor. An anchor can be:

    - a regex (`pattern_key`),
    - a single literal string (`literal_key`), or
    - a list of literal strings (`literal_key`), in which case any of these match

    All forms compile to a regex for matching, but we keep the literal strings around too: when present,
    the parser can emit regions faster, since it knows exactly how many trailing bytes might be a
    partial-match prefix (and that a finished literal match can never be extended by future bytes, except
    when a literal is a strict prefix of another in the same list).

    Returns `(compiled_re, literals, can_extend)`:

    - `compiled_re`: the compiled pattern.
    - `literals`: the literal strings, or `None` for a regex anchor.
    - `can_extend`: `True` iff one literal is a strict prefix of another, in which case a match at the
      buffer edge could still be lengthened by future input.
    z: cannot specify both 'z' and ''z: 'z(' list must contain at least one literalc              3  <   K   | ]  }t        |t                y wN)
isinstancer   .0ss     r$   	<genexpr>z"_compile_anchor.<locals>.<genexpr>e   s     7az!S)7s   z ' list must contain only stringsz+' must be a string or list of strings, got c              3  &   K   | ]	  }|d k(    yw) Nr"   r?   s     r$   rB   z"_compile_anchor.<locals>.<genexpr>j   s     )1qBw)s   z"' literals cannot be empty stringsT)keyreversec              3  \   K   | ]#  }D ]  }||k7  xr |j                  |        % y wr=   )
startswith)r@   abliteralss      r$   rB   z"_compile_anchor.<locals>.<genexpr>n   s2     XxX!a3ALLO3X3Xs   ),|c              3  F   K   | ]  }t        j                  |        y wr=   )reescaper?   s     r$   rB   z"_compile_anchor.<locals>.<genexpr>o   s     9A299Q<9s   !NFz
: invalid z regex: )NNF)
ValueErrorr>   r   listallr   fromkeystyper   anysortedlenjoinrN   compileDOTALLerror)
scopefieldliteral_keypattern_keyrawordered
can_extendpatternerK   s
            @r$   _compile_anchorre   I   s   & eu 4E7"9+gk]Z[\]]eK c3uHT" E7#k]:b!cdd7377 E7#k]:Z![\\DMM#./Hwc+6abfgjbkbtbtauvww)))wc+6XYZZsD9XXXX
((999zz'299-xCCe	R::eK0"))<dEII  xx 	RwjXaSIJPQQ	Rs   )E? ?F+F&&F+c                   t        | t              s!t        dt        |       j                         | j                  dd      }|dk7  rt        d|       t        |       h dz
  x}rt        dt        |             t        | j                  di       t              st        d      | j                  d	i       }t        |t              r|st        d
      y)zValidate the top-level response_template structure: it must be a dict of the expected version with
    only known keys, a dict of `defaults`, and a non-empty dict of `fields`. Per-field specs are checked
    separately in `_build_field`.z&response_template must be a dict, got versionr   z'Unsupported response_template version: >   r(   rg   r'   r-   r.   z#Unknown keys in response_template: r'   z)response_template.defaults must be a dictr(   z1response_template.fields must be a non-empty dictN)r>   r   rP   rT   r   getsetrV   )specrg   unknown_template_keys
fields_raws       r$   _validate_template_shaperm   y   s     dD!A$t*BUBUAVWXXhhy!$G!|B7)LMM #D	,u uuu>vF[?\>]^__dhhz2.5DEE(B'Jj$'zLMM 0:r#   c                   h d}d|  d}t        |t              st        | d      t        |      |z
  x}rt        | dt	        |             |j                  dd      }|t        vr t        | d| d	t	        t                     t        ||d
d      \  }}}t        ||dd      \  }	}
}|j                  d      }|j                  dd      }t        |t              s#t        | dt        |      j                         |r|t        | d      |t        ||       nZt               }||t        |j                        z  }|	|t        |	j                        z  }|rt        | dt	        |       d      t        | ||||	|
|||j                  di       |j                  dd      |j                  dd      ||      S )zKValidate a single field spec and compile it into a `ResponseTemplateField`.>
   opencloser   r   r   r   r   open_patternclose_patternr   zField 'r;   z must be a dictz: unknown keys r   r4   z: unknown content parser 'z'. Available: ro   rq   rp   rr   r   r   Fz%: transform_each must be a bool, got z5: transform_each is set but no transform was providedz5: open_pattern/close_pattern declares named group(s) z, but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.r   r   r   T)r   r   r   r   r   r   r   r   r   r   r   r   r   )r>   r   rP   ri   rV   rh   r	   re   r   rT   r   r
   
groupindexr   )r   r]   _ALLOWED_FIELD_KEYSr\   unknown_field_keysr   r   r   r   r   r   r   r   r   captured_namess                  r$   _build_fieldrw      s    dV1EeT"E7/233 Z*====E7/&9K2L1MNOOii	6*Go%E7"<WI^TZ[jTkSlmnn6EeUTZ\j6k3G]39HPUW^`o9p6Hn6		+&IYY/7Nnd+E7"G^H\HeHeGfghh)+E7"WXYY"5)4 c'"4"455Nc("5"566N'N.)* +<=  !# 7%!9YY~r2		)U+:t,% r#   c           	        t        | t              r| S t        |        | d   j                         D ci c]  \  }}|t	        ||       }}}|j                         D cg c]  \  }}|j
                  | }}}t        |      dkD  rt        ddj                  |             t        d| dd      \  }}}|t        d      t        t        | j                  d	i             ||r|d
   nd ||      S c c}}w c c}}w )Nr(   r   zpAt most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: z, response_templater-   r.   zGresponse_template must define 'start_anchor' or 'start_anchor_pattern'.r'   r   )r'   r(   r*   r)   r+   )r>   r&   rm   itemsrw   r   rW   rP   rX   re   r   rh   )	rj   r   r`   r(   r]   implicit_fieldsr)   r+   _s	            r$   load_response_templater}      s   $()T"=A(^=Q=Q=ST	cdLs++TFT/5||~WeAVtWOW
?a66:ii6P5QS
 	

 1@T>3I1-O*A bccdhhz2./'6#D'3  UWs   C7!C=6C=)r&   r   r}   )
r\   r   r]   r   r^   r   r_   r   r9   z"tuple[Any, list[str] | None, bool])rj   r   r9   None)r   r   r]   r   r9   r   )rj   zdict | ResponseTemplater9   r&   )__doc__
__future__r   dataclassesr   typingr   regexrN   utilsr   content_parsersr	   r
   
get_loggerr   r1   r   r&   re   rm   rw   r}   __all__r"   r#   r$   <module>r      s    C " !    H 
		H	%       2-`N$>B: Rr#   