Ë
    éÓ^j­G  ã            ]       ó  — d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
mZmZmZ ddlZddlmZmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ  G d„ d«      Zddddddddddddddddddddddddddeedddddddddddej:                   ej<                  d«      dddf,deeeeef   ef      dee
e      dee
e       deed f   d!ed"ed#e!d$ee   d%e!d&e d'e"d(ed)e!d*ee"   d+eee e f      d,eee e f      d-e d.e d/e d0ee    d1e d2e d3ee"   d4ed5ed6e"d7ee d f   d8ee d f   d9ee    d:ee"   d;ee   d<ed=e!d>ed?ed@edAedBe!dCe!dDejF                  dEee"ej<                  f   dFe!dGe"dHe!dIeejH                  jJ                  jL                  ef   fZdJ„Z'y)Ka¸  NaFlex data loader for dynamic sequence length training.

This module provides a specialized data loader for Vision Transformer models that supports:
- Dynamic sequence length sampling during training for improved efficiency
- Variable patch size training with probabilistic selection
- Patch-level random erasing augmentation
- Efficient GPU prefetching with normalization

Hacked together by / Copyright 2025, Ross Wightman, Hugging Face
é    N)Úsuppress)Úpartial)ÚCallableÚDictÚIteratorÚListÚOptionalÚTupleÚUnioné   )ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú_worker_initÚadapt_to_chs)ÚNaFlexMapDatasetWrapperÚNaFlexCollator)ÚPatchRandomErasing)Úcreate_transformc                   óv  — e Zd ZdZeed ej                  d«      dddddd	f
d
ej                  j                  j                  deedf   deedf   dedej                  deej                     dedededededdfd„Zdeeeeej*                  f   ej*                  f      fd„Zdefd„Zed„ «       Zed„ «       Zy)ÚNaFlexPrefetchLoaderz;Data prefetcher for NaFlex format which normalizes patches.é   ÚcudaNç        Úconstr   r   TÚloaderÚmean.ÚstdÚchannelsÚdeviceÚ	img_dtypeÚre_probÚre_modeÚre_countÚre_num_splitsÚpatchify_channels_lastÚreturnc                 óú  — || _         || _        |xs t        j                  | _        t        ||«      }t        ||«      }|| _        || _        |rdd|fnd|df}t        j                  |D �cg c]  }|dz  ‘Œ	 c}|| j                  ¬«      j                  |«      | _
        t        j                  |D �cg c]  }|dz  ‘Œ	 c}|| j                  ¬«      j                  |«      | _        |dkD  rt        |||	|
|¬«      | _        nd| _        |j                  dk(  xr t        j                  j!                  «       | _        |j                  dk(  xr t        j$                  j!                  «       | _        yc c}w c c}w )	a“  Initialize NaFlexPrefetchLoader.

        Args:
            loader: DataLoader to prefetch from.
            mean: Mean values for normalization.
            std: Standard deviation values for normalization.
            channels: Number of image channels.
            device: Device to move tensors to.
            img_dtype: Data type for image tensors.
            re_prob: Random erasing probability.
            re_mode: Random erasing mode.
            re_count: Maximum number of erasing rectangles.
            re_num_splits: Number of augmentation splits.
            patchify_channels_last: Per-patch flat layout produced by the
                upstream Patchify. ``True`` (default): channel index is the
                innermost (P-P-C). ``False``: channel index is the outermost
                (C-P-P). Controls how ``patches`` tensors are viewed for
                normalization and erasing.
        r   éÿ   )r   Údtyper   )Ú
erase_probÚmodeÚ	max_countÚ
num_splitsr   Nr   Únpu)r   r   ÚtorchÚfloat32r    r   r   r%   ÚtensorÚviewr   r   r   Úrandom_erasingÚtyper   Úis_availableÚis_cudar.   Úis_npu)Úselfr   r   r   r   r   r    r!   r"   r#   r$   r%   Únormalization_shapeÚxs                 úb/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/data/naflex_loader.pyÚ__init__zNaFlexPrefetchLoader.__init__   sU  € ðB ˆŒØˆŒØ"Ò3¤e§m¡mˆŒô ˜D (Ó+ˆÜ˜3 Ó)ˆØ ˆŒØ&<ˆÔ#Ù2H˜q ! XÑ.ÈqÐRZÐ\]ÐN^ÐÜ—L‘LØ"Ö#˜ˆQ�‹WÒ#¨F¸$¿.¹.ôJßJNÉ$ÐObÓJcð 	Œ	ä—<‘<Ø!Ö"˜ˆQ�‹WÒ"¨6¸¿¹ôIßIMÉÐNaÓIbð 	Œð �RŠ<Ü"4Ø"ØØ"Ø(Øô#ˆDÕð #'ˆDÔð —{‘{ fÑ,ÒJ´·±×1HÑ1HÓ1JˆŒØ—k‘k UÑ*ÒG¬u¯y©y×/EÑ/EÓ/Gˆ�ùò# $ùâ"s   Á-E3Â4E8c           
   #   óp  K  — d}| j                   rPt        j                  j                  | j                  ¬«      }t        t        j                  j                  |¬«      }nd| j                  rPt        j                  j                  | j                  ¬«      }t        t        j                  j                  |¬«      }nd}t        }| j                  D �]N  \  }} |«       5  |j                  «       D ]W  \  }}t        |t        j                  «      sŒ!|dk(  r| j                  nd}||   j                  | j                  d|¬«      ||<   ŒY |j                  | j                  d¬«      }|d   }	|	j                   }
|	j"                  dk(  rP|
\  }}}| j$                  r|	j'                  ||d	| j(                  «      }nñ|	j'                  ||| j(                  d	«      }nÒ|	j"                  d
k(  rª|
dd \  }}| j$                  rK|
d	   | j(                  k(  sJ d| j(                  › d|
d	   › �«       ‚|	j'                  ||d	| j(                  «      }nd|
d   | j(                  k(  sJ d| j(                  › d|
d   › �«       ‚|	j'                  ||| j(                  d	«      }nt+        d|	j"                  › d�«      ‚|j-                  | j.                  «      j1                  | j2                  «      }| j4                  �| j$                  s |j7                  dd	«      j9                  «       }| j5                  ||d   |j;                  dd«      ¬«      }| j$                  s |j7                  dd	«      j9                  «       }|j'                  |
«      |d<   ddd«       |sf–— nd}|�‹| j                   r:t        j                  j=                  | j                  ¬«      j?                  |«       nE| j                  r9t        j                  j=                  | j                  ¬«      j?                  |«       |}|}�ŒQ f–— y# 1 sw Y   Œ¯xY w­w)zœIterate through the loader with prefetching and normalization.

        Yields:
            Tuple of (input_dict, targets) with normalized patches.
        T)r   )ÚstreamNÚpatches)r   Únon_blockingr)   )r   r@   r   éÿÿÿÿé   é   z	Expected z channels, got z&Unexpected patches tensor dimensions: z. Expected 3 or 5.éþÿÿÿÚpatch_coordÚpatch_valid)rE   rF   F) r6   r/   r   ÚStreamr   r   r>   r7   r.   r   r   ÚitemsÚ
isinstanceÚTensorr    ÚtoÚshapeÚndimr%   r2   r   Ú
ValueErrorÚsubr   Údivr   r3   Ú	transposeÚ
contiguousÚgetÚcurrent_streamÚwait_stream)r8   Úfirstr>   Ústream_contextÚnext_input_dictÚnext_targetÚkÚvr)   Úpatches_tensorÚoriginal_shapeÚ
batch_sizeÚnum_patchesÚ_r?   Ú
input_dictÚtargets                    r;   Ú__iter__zNaFlexPrefetchLoader.__iter__`   sõ  è ø€ ð ˆØ�<Š<Ü—Z‘Z×&Ñ&¨d¯k©kÐ&Ó:ˆFÜ$¤U§Z¡Z×%6Ñ%6¸vÔF‰NØ�[Š[Ü—Y‘Y×%Ñ%¨T¯[©[Ð%Ó9ˆFÜ$¤U§Y¡Y×%5Ñ%5¸fÔE‰NàˆFÜ%ˆNà,0¯K©Kó L	!Ñ(ˆO˜[ÙÓ!ñ =Jà+×1Ñ1Ó3ò ‘D�A�qÜ! !¤U§\¡\Õ2Ø23°y². §¢Àd˜Ø-<¸QÑ-?×-BÑ-BØ#'§;¡;Ø)-Ø"'ð .Có .˜¨Ò*ðð *Ÿn™n°D·K±KÈd˜nÓS�ð
 "1°Ñ!;�Ø!/×!5Ñ!5�à!×&Ñ&¨!Ò+à1?Ñ.�J ¨QØ×2Ò2à"0×"5Ñ"5°jÀ+ÈrÐSW×S`ÑS`Ó"a™ð #1×"5Ñ"5°jÀ+ÈtÏ}É}Ð^`Ó"a™Ø#×(Ñ(¨AÒ-à.<¸R¸aÐ.@Ñ+�J Ø×2Ò2à-¨bÑ1°T·]±]ÒBð [Ø'¨¯© °oÀnÐUWÑFXÐEYÐZó[ÐBà"0×"5Ñ"5°jÀ+ÈrÐSW×S`ÑS`Ó"a™ð  .¨aÑ0°D·M±MÒAð ZØ'¨¯© °oÀnÐUVÑFWÐEXÐYóZÐAà"0×"5Ñ"5°jÀ+ÈtÏ}É}Ð^`Ó"a™ä$Ð'MÈn×NaÑNaÐMbÐbtÐ%uÓvÐvð "Ÿ+™+ d§i¡iÓ0×4Ñ4°T·X±XÓ>�à×&Ñ&Ð2ð  ×6Ò6Ø")×"3Ñ"3°B¸Ó";×"FÑ"FÓ"H˜Ø"×1Ñ1ØØ$3°MÑ$BØ$3×$7Ñ$7¸ÀtÓ$Lð 2ó �Gð
  ×6Ò6Ø")×"3Ñ"3°B¸Ó";×"FÑ"FÓ"H˜ð .5¯\©\¸.Ó-I� 	Ñ*÷{=Jñ~ Ø  &Ð(Ó(à�àÐ!Ø—<’<Ü—J‘J×-Ñ-°T·[±[Ð-ÓA×MÑMÈfÕUØ—[’[Ü—I‘I×,Ñ,°D·K±KÐ,Ó@×LÑLÈVÔTà(ˆJØ ŠFðYL	!ð\ ˜&Ð Ó ÷[=Jð =Jüs&   ‚CP6Ã1P*ÄI+P*Í<B.P6Ð*P3	Ð/P6c                 ó,   — t        | j                  «      S )zhGet length of underlying loader.

        Returns:
            Number of batches in the loader.
        )Úlenr   ©r8   s    r;   Ú__len__zNaFlexPrefetchLoader.__len__Á   s   € ô �4—;‘;ÓÐó    c                 ó.   — | j                   j                  S )zrGet sampler from underlying loader.

        Returns:
            Sampler from the underlying DataLoader.
        )r   Úsamplerrf   s    r;   rj   zNaFlexPrefetchLoader.samplerÉ   ó   € ð �{‰{×"Ñ"Ð"rh   c                 ó.   — | j                   j                  S )zrGet dataset from underlying loader.

        Returns:
            Dataset from the underlying DataLoader.
        )r   Údatasetrf   s    r;   rm   zNaFlexPrefetchLoader.datasetÒ   rk   rh   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r/   r   ÚutilsÚdataÚ
DataLoaderr
   ÚfloatÚintr	   r)   ÚstrÚboolr<   r   r   rJ   rc   rg   Úpropertyrj   rm   © rh   r;   r   r      sY  „ ÙEð
 '<Ø%9ØØ#/ 5§<¡<°Ó#7Ø/3ØØ"ØØ!"Ø+/ñ@Hà—K‘K×$Ñ$×/Ñ/ð@Hð ˜˜s˜
Ñ#ð@Hð �u˜c�zÑ"ð	@Hð
 ð@Hð —L‘Lð@Hð   §¡Ñ,ð@Hð ð@Hð ð@Hð ð@Hð ð@Hð %)ð@Hð ó@HðD_!˜( 5¨¨c°5·<±<Ð.?Ñ)@À%Ç,Á,Ð)NÑ#OÑPó _!ðB ˜ó  ð ñ#ó ð#ð ñ#ó ñ#rh   r   )é€   é   é@  i  i   r}   é    Fr   r   g      à?gš™™™™™Ù?Úbilinearé   é*   Tr   ÚallÚ
patch_sizeÚpatch_size_choicesÚpatch_size_choice_probsÚtrain_seq_lens.Úmax_seq_lenr^   Úis_trainingÚmixup_fnÚno_augr!   r"   r#   Úre_splitÚtrain_crop_modeÚscaleÚratioÚhflipÚvflipÚcolor_jitterÚcolor_jitter_probÚgrayscale_probÚgaussian_blur_probÚauto_augmentÚnum_aug_repeatsÚnum_aug_splitsÚinterpolationr   r   Úcrop_pctÚ	crop_modeÚcrop_border_pixelsÚnum_workersÚdistributedÚrankÚ
world_sizeÚseedÚepochÚuse_prefetcherÚ
pin_memoryr    r   Úpersistent_workersÚworker_seedingr%   r&   c-                 ó’  — |�r|dk(  sJ d«       ‚t        t        fi dd“d|	“d|“d|“d|“d	|“d
|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|
“d|“d|“d|&“dd“d|,“Ž}-t        |«      }.||.z  }/t        | t        j
                  j                  j                  «      rJ d«       ‚t        | |-|||||/||$|!|"|#d|%|,¬«      }0t        j
                  j                  j                  |0dd| d|'t        t        |+¬ «      |*¬!«      }1|&rt        |1|||(|)|
|||,¬"«	      }1|1S t        d||||&d||d|,¬#«
      | _        t        |¬$«      }2d}3|!r<t        | t        j
                  j                  j                  «      sdd%lm}4  |4| «      }3t        j
                  j                  j                  | |d| |3|2|'d¬&«      }1|&rt        |1|||(|)|,¬'«      }1|1S )(ux
  Create a data loader with dynamic sequence length sampling for training.

    Args:
        dataset: Dataset to load from.
        patch_size: Single patch size to use.
        patch_size_choices: List of patch sizes for variable patch size training.
        patch_size_choice_probs: Probabilities for each patch size choice.
        train_seq_lens: Training sequence lengths for dynamic batching.
        max_seq_len: Fixed sequence length for validation.
        batch_size: Batch size for validation and max training sequence length.
        is_training: Whether this is for training (enables dynamic batching).
        mixup_fn: Optional mixup function.
        no_aug: Disable augmentation.
        re_prob: Random erasing probability.
        re_mode: Random erasing mode.
        re_count: Maximum number of erasing rectangles.
        re_split: Random erasing split flag.
        train_crop_mode: Training crop mode.
        scale: Scale range for random resize crop.
        ratio: Aspect ratio range for random resize crop.
        hflip: Horizontal flip probability.
        vflip: Vertical flip probability.
        color_jitter: Color jitter factor.
        color_jitter_prob: Color jitter probability.
        grayscale_prob: Grayscale conversion probability.
        gaussian_blur_prob: Gaussian blur probability.
        auto_augment: AutoAugment policy.
        num_aug_repeats: Number of augmentation repeats.
        num_aug_splits: Number of augmentation splits.
        interpolation: Interpolation method.
        mean: Normalization mean values.
        std: Normalization standard deviation values.
        crop_pct: Crop percentage for validation.
        crop_mode: Crop mode.
        crop_border_pixels: Crop border pixels.
        num_workers: Number of data loading workers.
        distributed: Whether using distributed training.
        rank: Process rank for distributed training.
        world_size: Total number of processes.
        seed: Random seed.
        epoch: Starting epoch.
        use_prefetcher: Whether to use prefetching.
        pin_memory: Whether to pin memory.
        img_dtype: Image data type.
        device: Device to move tensors to.
        persistent_workers: Whether to use persistent workers.
        worker_seeding: Worker seeding mode.
        patchify_channels_last: Per-patch flat layout. ``True`` (default):
            channel index varies fastest (NaFlex default). ``False``: C-P-P
            (channels first â€” Gemma4 / HF native layout). Forwarded to the
            upstream ``Patchify`` and to the prefetcher's normalization layout.

    Returns:
        DataLoader or NaFlexPrefetchLoader instance.
    r   z=Augmentation repeats not currently supported in NaFlex loaderrˆ   TrŠ   rŒ   r�   rŽ   r�   r�   r‘   r’   r“   r”   r•   r˜   r   r   r™   rš   r›   r!   r"   r#   r¢   Únaflexr%   Fz IterableDataset Wrapper is a WIP)Útransform_factoryrƒ   r„   r…   Úseq_lensÚmax_tokens_per_batchr‰   r    r�   rž   rŸ   Úshuffler¡   r%   N)r¥   )r^   r«   rœ   rj   r£   Úworker_init_fnr¤   )r   r   r    r   r!   r"   r#   r%   )
rˆ   r˜   r   r   r¢   r§   rƒ   r‡   Úpatchifyr%   )r‡   )ÚOrderedDistributedSampler)r^   r«   rœ   rj   Ú
collate_fnr£   Ú	drop_last)r   r   r    r   r%   )r   r   ÚmaxrI   r/   rr   rs   ÚIterableDatasetr   rt   r   r   Ú	transformr   Útimm.data.distributed_samplerr®   )5rm   rƒ   r„   r…   r†   r‡   r^   rˆ   r‰   rŠ   r!   r"   r#   r‹   rŒ   r�   rŽ   r�   r�   r‘   r’   r“   r”   r•   r–   r—   r˜   r   r   r™   rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r£   r    r   r¤   r¥   r%   r¨   Úmax_train_seq_lenrª   Únaflex_datasetr   r¯   rj   r®   s5                                                        r;   Úcreate_naflex_loaderr·   Ü   sÈ  € òR à !Ò#ÐdÐ%dÓdÐ#ä#Üò
áð
ñ ð
ñ ,ð	
ñ
 ð
ñ ð
ñ ð
ñ ð
ñ &ð
ñ 0ð
ñ *ð
ñ  2ð
ñ &ð
ñ (ð
ñ ð
ñ  ð!
ñ" ð#
ñ$  ð%
ñ&  2ð'
ñ( ð)
ñ* ð+
ñ, ð-
ñ. *ð/
ñ0 ð1
ñ2 $:ð3
Ðô8   Ó/ÐØ)Ð,=Ñ=Ðä�gœuŸ{™{×/Ñ/×?Ñ?Ô@Ø<Ð<Ó<�5ä0ØØ/Ø!Ø1Ø$;Ø#Ø!5ØØØ#ØØ!ØØØ#9ô
ˆô& —‘×!Ñ!×,Ñ,ØØØØ#ØØ!Ü"¤<ÀÔOØ1ð -ó 	
ˆñ Ü)ØØØØ#ØØØØ!Ø'=ô
ˆFðv €Mô[ -ØØ'ØØà)ØØ!Ø#ØØ#9ô
ˆÔô $°Ô<ˆ
ð ˆÙœz¨'´5·;±;×3CÑ3C×3SÑ3SÔTåOÙ/°Ó8ˆGä—‘×!Ñ!×,Ñ,ØØ!ØØ#ØØ!Ø!Øð -ó 	
ˆñ Ü)ØØØØ#ØØ'=ôˆFð €Mrh   )(rq   ÚmathÚ
contextlibr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r/   Ú	constantsr   r   r   r   r   r¶   r   r   Únaflex_random_erasingr   Útransforms_factoryr   r   r0   r   rv   ru   rx   rw   r)   rr   rs   rt   r·   rz   rh   r;   ú<module>r¿      s’  ðñ	ó Ý Ý ß I× IÑ Ió ç Bß .ß CÝ 5Ý 0÷~#ñ ~#ðF =AØ26Ø9=Ø*DØØØ!Ø'+àØØØØØ)-Ø/3Ø/3ØØØ!Ø-1Ø "Ø$&Ø&*Ø ØØ'Ø"7Ø!5Ø$(Ø#'Ø,0àØ!ØØØØØ#ØØ!&§¡Ø+7¨5¯<©<¸Ó+?Ø#'Ø#Ø'+ñ_ià˜U 5¨¨c¨¡?°CÐ#7Ñ8Ñ9ðið % T¨#¡YÑ/ðið "*¨$¨u©+Ñ!6ð	ið
 ˜c 3˜h™ðið ðið ðið ðið ˜8Ñ$ðið ðið ðið ðið ðið ðið  " #™ð!ið" ˜˜e U˜lÑ+Ñ,ð#ið$ ˜˜e U˜lÑ+Ñ,ð%ið& ð'ið( ð)ið* ð+ið, $ E™?ð-ið. ð/ið0 "ð1ið2 ˜s‘mð3ið4 ð5ið6 ð7ið8 ð9ið: �E˜3�JÑð;ið< �5˜#�:Ñð=ið> ˜5‘/ð?ið@ ˜C‘=ðAiðB % S™MðCiðF ðGiðH ðIiðJ ðKiðL ðMiðN ðOiðP ðQiðR ðSiðT ðUiðV —;‘;ðWiðX �c˜5Ÿ<™<Ð'Ñ(ðYiðZ !ð[ið\ ð]ið^ !%ð_ið` 
ˆu�{‰{×Ñ×*Ñ*Ð,@Ð@Ñ	Aôairh   