
    ^ja                        d dl Z d dlmZ d dlZd dlmZ d dlmc m	Z
 d dlmc mZ d dlmZ d dlmZ d dlmZmZmZmZ d dlmZmZ d dlmZ d dlmZmZmZ d d	lm Z  d d
l!m"Z" ejF                  jH                  Z$dgZ%e jL                  d        Z'de(e"df   de)de)fdZ*de(e"df   dedefdZ+dejX                  jZ                  de(e.df   de/e0e.f   defdZ1d Z2dejX                  jZ                  de(e.df   de/e0e.f   de.fdZ3dejX                  jZ                  de(e.df   de/e0e.f   de.fdZ4dedededz  dedz  de)d e)d!ejj                  d"e)ded#e)de(eef   fd$Z6dejX                  jZ                  de(e.df   de/e0e.f   de.fd%Z7d&ededededz  de)d e)d'ed!ejj                  d"e)ded#e)defd(Z8dejX                  jZ                  de(e.df   de/e0e.f   de.fd)Z9e$jd                  jt                  e3e$jv                  jt                  e4e$jx                  jt                  e7e$jz                  jt                  e7e$j|                  jt                  e9e$j~                  jt                  e9iZ@d* ZAd+ ZBy),    N)cast)Tensor)
DeviceMesh)DTensorPartial	ReplicateShard)DTensorSpec
TensorMeta)_MaskPartial)	_skip_dim	Reductionreplicate_reduction_dims)normalize_dim)	Placementloss_parallelc               #   <   K   t                d t                yw)uz  
    A context manager that enables loss parallelism, where efficient parallelized loss computation
    can be performed when the input is sharded on the class dimension. Currently only the cross-entropy
    loss is supported.

    Within this context manager, one can use :func:`~torch.nn.functional.cross_entropy` or
    :class:`~torch.nn.CrossEntropyLoss` as usual, with the following assumptions on the input parameters.
    The corresponding ``backward()`` call, if any, also needs to happen under this context manager.

    Both one-dimensional and multi-dimensional ``DeviceMesh`` are supported. On a
    multi-dimensional mesh, exactly one mesh dim must shard the class dimension —
    that dim is treated as the "TP" dim and can be in any position (not necessarily
    last). The remaining mesh dims may be ``Shard`` (e.g. batch-sharded for DP/CP)
    or ``Replicate``.

    Args:
        input (:class:`DTensor`):
            Input logits. Assumed to be sharded on the class dimension on exactly
            one mesh dim.
        target (Union[:class:`torch.Tensor`, :class:`DTensor`]):
            Must be ground truth class indices (class probabilities currently not
            supported). Its placements are derived from ``input``'s placements by
            (1) replacing the TP dim's ``Shard(class_dim)`` with ``Replicate`` and
            (2) shifting any other ``Shard(d)`` with ``d > class_dim`` down by one
            (since the class tensor dim is removed from the output)::

                input tensor shape       input placements             target shape      target placements
                ----------------------   --------------------------   ---------------   ------------------------
                (batch, class)           (Shard(0), Shard(1))         (batch,)          (Shard(0), Replicate())
                (batch, class)           (Replicate(), Shard(1))      (batch,)          (Replicate(), Replicate())
                (batch, class, seq)      (Shard(2), Shard(1))         (batch, seq)      (Shard(1), Replicate())
                (batch, class, seq)      (Shard(0), Shard(1))         (batch, seq)      (Shard(0), Replicate())

            A plain :class:`torch.Tensor` is only accepted when the derived target
            placements are all ``Replicate`` (e.g. on a one-dimensional mesh);
            otherwise a :class:`DTensor` must be passed explicitly so that the
            intended sharding is unambiguous.
        weight (Union[:class:`torch.Tensor`, :class:`DTensor`], optional):
            If given, assumed to be replicated across the ``DeviceMesh``.
        label_smoothing:
            Currently not supported.

    Returns:
        A :class:`DTensor` whose placements depend on ``reduction``:

        - ``reduction="none"`` — per-sample loss; inherits the ``target``
          placements (``Shard`` on batch-parallel dims, ``Replicate`` on TP).
        - ``reduction="sum"`` — scalar; ``Replicate`` on the TP dim (the custom
          all-reduce makes each rank's local value globally correct over TP),
          ``Partial("sum")`` on every other ``Shard`` mesh dim so that the
          cross-rank reduction happens lazily on materialization/redistribution,
          and ``Replicate`` on ``Replicate`` mesh dims.
        - ``reduction="mean"`` — only supported on a one-dimensional mesh; returns
          a fully replicated :class:`DTensor`.

        On a one-dimensional mesh all three cases simplify to a fully replicated
        :class:`DTensor` (the original behavior).

    .. note::
        ``reduction="mean"`` is only supported on a one-dimensional ``DeviceMesh``.
        On a multi-dimensional mesh the per-rank division by ``total_weight`` uses a
        local count, so aggregating across non-TP dims does not yield the correct
        global mean; use ``reduction="sum"`` or ``"none"`` instead and divide by the
        global count yourself if needed.

    Example:
        A sharded DTensor is manually created here to showcase the usage.
        In practice, it is usually the output of a TP module.

        >>> # xdoctest: +SKIP("distributed")
        >>> from torch.distributed.tensor.parallel import loss_parallel
        >>> from torch.distributed.device_mesh import init_device_mesh
        >>> ...
        >>> device_mesh = init_device_mesh("cuda", (8,))
        >>> input = torch.randn(4, 16, device="cuda", requires_grad=True)
        >>> dist_input = distribute_tensor(input, device_mesh, placements=[Shard(1)])
        >>> target = torch.randint(16, (4,), device="cuda")
        >>> with loss_parallel():
        >>>     loss = F.cross_entropy(dist_input, target, reduction="mean")
        >>>     loss.backward()
        >>> ...
    N)_enable_custom_loss_ops_disable_custom_loss_ops     q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/tensor/parallel/loss.pyr   r      s     h 	   
placements.dimreturnc           
      j   t        |       D cg c]  \  }}|j                  |      s| }}}t        |      dk7  rt        d| d|  dt        |       d      |d   }t        |       D ]>  \  }}||k(  r|j                         r|j	                         r.t        d| d| d	       |S c c}}w )
N   zHloss_parallel() requires exactly one mesh dim to shard tensor dimension z, but got placements z with z such mesh dim(s).r   zGloss_parallel() expects non-TP mesh dims to be Shard or Replicate, got z at dim .)	enumerateis_shardlen
ValueErroris_replicate)r   r   ipshard_mesh_dimsmesh_dims         r   _find_all_reduce_mesh_dimr)   ~   s    %.z%:NTQajjoqNON
?q 2:,f?#$$68
 	

 q!H*% 1=

 0++,#XaS; 	 O! Os
   B/B/meshc                 R   t        | t              r-| j                  |k(  r| S t        d| d| j                   d      t        | t        j
                        r:t        d |D              rt        d| d      t        j                  | ||d      S t        d	t        |              )
Nz	Expected z	 but got r   c              3   <   K   | ]  }|j                           y wN)r!   ).0r&   s     r   	<genexpr>z#_cast_to_dtensor.<locals>.<genexpr>   s     0qzz|0r   zZloss_parallel() requires a DTensor (not a plain torch.Tensor) when the derived placements u    contain Shard — this happens on a multi-dimensional mesh with a batch-sharded non-TP dim. Wrap the tensor with DTensor.from_local or distribute_tensor to make the sharding explicit.F)device_meshr   	run_checkzUnsupported type )
isinstancer   r   RuntimeErrortorchr   anyr#   
from_local	TypeErrortype)tensorr   r*   s      r   _cast_to_dtensorr:      s     &'"
*M:,i@Q@Q?RRSTUU	FELL	)0Z00 //9l ;12  !!u
 	
 +DL>:;;r   op_callargskwargsc                 V   t         j                  j                  | ||      }|j                  t	        d      t         j                  j
                  j                  |j                        }t        |t              r|S t        |t              r|d   S t        dt        |       d      )Nz9op_info.schema should not be None after unwrap_to_op_infor   zUnexpected tensor meta type: r   )r   _op_dispatcherunwrap_to_op_infoschemaAssertionErrorsharding_propagator_propagate_tensor_metar2   r   tupler3   r8   )r;   r<   r=   op_infotensor_metas        r   rD   rD      s    
 $$66wfMG~~G
 	
 ((<<SSK +z*	K	'1~:4;L:MQOPPr   c                    |r#| j                   t        j                  k7  rt        t	        j
                  | t        j                  j                        \  }}| j                  |t        j                        } | j                         dk(  r| }nYt        j                  | |d      }t        j                  |t        j                  j                   j"                  ||f      }| |z
  }t        j$                  t        j&                  |      |d      }	t        j                  |	t        j                  j(                  j"                  ||f      }	t        j*                  |	      }
||
z
  }|s|j                  |      }|S )N)type_promotion_kind)dtypememory_formatr   T)keepdim)reduceOpgroup)rJ   r4   halfrB   utilselementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTtocontiguous_formatnumelamaxfuncol
all_reducec10dReduceOpMAXnamesumexpSUMlog)xr   half_to_floatr*   r(   computation_dtyperesult_dtypeshiftedx_maxshifted_sumexpshifted_logsumexpresults               r   _log_softmaxrk      s1   77ejj   &+&>&>	uDDLL'#| 	
$E4K4KLAwwyA~

1c40!!DMM--224:J
 e)YYuyy13EN&&!2!2!7!7h?ON 		.1((F<(Mr   c                    t        t        |d         }t        t        |d         }t        t        |d         }|j                  }t        ||j                               }t        |j                  |      }t        | ||      }t        |j                  |||j                  |      }	t        |j                  |j                  |      }
t        |	|
|	j                        S )Nr   r      rG   requires_grad)r   r   intbool_specr   r   r)   r   rD   rk   _local_tensorr*   r
   rp   )r;   r<   r=   rb   r   rc   specr(   output_tensor_metaresres_specs              r   _log_softmax_handlerry      s    
 	Wd1gA
sDG
CtAw'M77D
QUUW
%C(#>H/vF
q]DIIx
PC		&H '' r   c                     t        t        |d         }t        t        j                  |d         }|j	                  |      S )Nr      )r   r   r4   rJ   rT   )r;   r<   r=   grad_outputinput_dtypes        r   _log_softmax_backward_handlerr~     s7    
 wQ(Ku{{DG,K>>+&&r   rb   targetweightlocal_weight	reductionignore_indexinput_shapechannel_dimr(   c
                 X   | j                         ddk  rddt        dt        ffd}
| |
|      }|t         |
|      }| |z  } t        j                  ||k7  |d      }|j                        }t        |      }|j                  |||	      }t        j                  | |      }|j                  |||	      }|j                         }t        j                  ||k7  |d      }|t        j                  j                  k(  rdkD  r| j                  dd	      }||fS ||t        | j                         }d
|<   j#                  |      }t        j                  ||      j                        }t        j                  ||k7  |d      }|j%                         }n"||k7  j%                         j'                  |       }|t        j(                  j                  k(  r|j%                         }||fS |t        j*                  j                  k(  r|j%                         |z  }||fS )Nr   rm   r   r   r   c                 l    dkD  r+dgz  }| j                   d   |<   | j                  |      }|S | }|S )Nr   r   )shapeview)r   r   wr   n_dimss      r   _weight_viewz'_nll_loss_forward.<locals>._weight_view   sQ    A:E "(aE+E"A  Ar   offset_shape
offset_dimr   g        )r   r   rB   r4   where	unsqueezer   _partition_valuegather_reduce_valuesqueezer   NONEvaluenew_fulllistr   expandr^   rT   r`   MEAN)rb   r   r   r   r   r   r   r   r*   r(   r   r   local_wsafe_targetsafe_target_partial_placementsafe_target_partial_result_partialresult_reducedrj   total_weight	new_shapewsumr   s          `               @r   _nll_loss_forwardr     s    UUWFKz	V 	 	    |,K++f4fa@K((5L %++V,==dH \\![2FGN&44^T8TN$$[11F[[</;FINN(((VaZzz"c*|##M	!#	+HHY||A{L9AA+N{{6\14;xxz,.33588; IMM''' < 
inn**	*,<r   c                 p   t        t        |d         }|d   }|d   }t        t        |d         }t        t        |d         }|j                         dk\  rdnd}|j                  }	t        |	j                  |      }
t        t        |	j                  |g      |      }t               f|	j                  j                  z  }t        |||	j                        }d }|t        |||	j                        }t        |	j                  j                        D cg c]  }||
k(  rt        d      n	t                }}|j                  |	j                  |      j                   }|j"                  d   |j                   j"                  |   k7  rt$        |t&        j(                  j*                  k(  r|}n|t&        j,                  j*                  k(  r<|	j                  j                  dkD  r#t/        d|	j                  j                   d      g }t1        |	j                        D ]_  \  }}||
k(  r|j3                  t                      %|j5                         r|j3                  t7                      O|j3                  |       a t9        |      }t;        |      }||c|d<   |d<   t=        | t9        |      |      }t?        |j                   |j                   ||j                   nd ||||j"                  ||	j                  |

      \  }}tA        |	j                  ||      }t        |||jB                  	      |fS c c}w )
Nr   r   rm   r{      zjloss_parallel() with reduction='mean' is only supported on one-dimensional DeviceMesh; got mesh with ndim=z(. Use reduction='sum' or 'none' instead.rn   ro   )"r   r   rq   r   rs   r)   r   r   r   r   r*   ndimr:   ranger	   redistributert   r   rB   r   r   r   r   NotImplementedErrorr    appendr!   r   rE   r   rD   r   r
   rp   )r;   r<   r=   rb   r   r   r   r   r   ru   r(   target_placementsall_replicate_placementsr   r%   sharded_placementsoutput_placementsout_placements_listr&   rv   rj   r   out_specs                          r   _nll_loss_forward_handlerr   ]  s   
 	Wd1gA!WF!WFS$q'"IT!W%Luuw!|!K77D(+FH " ;-@+ !*~		>f&7CFL!&*BDIIN
 AFdiinn@U
;<XE!H9;6
 
 **4996HIWWa AOO$9$9+$FF  INN(((- 	,,,!1C%B99>>""JL  02doo. 	.DAqH}#**9;7#**795#**1-	. ""56 :DvDGT!W/tfM,	 & 2			FL 499&7EWXH 	 ..	
 	
 
m
s   !L3r|   r   c                    |j                         dk  rdnd}|t        j                  j                  k(  r| |z  } |j	                  |      }t        j                  ||k7  |d      }t        j                  |      }t        ||      }|j                  |      j                         }|j                  ||	|
      }|j                  j                  t        |j                  j                  j                  |j                         dz
  }t        j"                  |j$                  d   |j&                        }|j                         dk(  r|||<   n|j                         dk(  r||||f<   ne|j)                  |d      }|j$                  }|j+                  d|j$                  |         }||||f<   |j-                  |      j)                  |d      }|j                         | j                         cxkD  rdkD  rn n| j	                  |      } |t/        |j                               D cg c]  }d }}|j$                  d   ||<   |j+                  |      }t1        |j$                        }d||<   |j3                  |      }t        j4                  |||      }| |z  } t        j                  ||k7  | d      } |t        j6                  |      z   | z  S c c}w )Nrm   r   r   r   g      ?)devicer   )r   r   r   r   r   r4   r   
zeros_liker   r   flattenr   mask_bufferdatarB   rT   rJ   aranger   r   	transposereshaper   r   r   r   r   r_   )r|   rb   r   r   r   r   r   r   r   r*   r(   r   
grad_inputr   masked_safe_targetgrad_update	arange_1dgrad_input_tintermidate_shapegrad_input_2d_r   r   w_targets                           r   "_nll_loss_and_log_softmax_backwardr     s    uuw{!KINN(((!L0k*F++f4fa@K!!!$J %++V%%k2::<K*;;KxX$$))1#//4477
8H8HICOK  #,>,E,EI
 	uuw!|)4
%&	
A4?
9001!++K<(..$,,R1EF7Bi!334"''(9:DD[RTU
~~+//+/a/!++K8 %aeeg/1Q/	/!'a	+	* M	!#	+MM)$<<;7!H,++f4k1EK 1%44# 0s   &	Kc                 R   t        t        |d         }t        t        |d         }|d   }|d   }t        t        |d         }t        t        |d         }t        t        |d         }	|j	                         dk\  rdnd}
|j
                  }t        |j                  |
      }t        t        |j                  |
g      |
      }t               f|j                  j                  z  }t        |||j                        }|t        |||j                        }|t        j                  j                   k(  r|j#                  |j                  |      }t%        |      }||d<   ||c|d<   |d<   t        |	||j                        |d<   t'        | t)        |      |      }t+        |j,                  |j,                  |j,                  ||j,                  nd |||	|j.                  |
|j                  |      }t1        |j                  |j                  |      }t        |||j2                  	      S )
Nr   r   rm   r{   r         rn   ro   )r   r   rq   r   r   rs   r)   r   r   r   r   r*   r   r:   r   r   r   r   r   rD   rE   r   rt   r   r
   rp   )r;   r<   r=   r|   rb   r   r   r   r   r   r   ru   r(   r   r   rv   rj   r   s                     r   _nll_loss_backward_handlerr   	  s   
 wQ(KWd1gA!WF!WFS$q'"IT!W%LQ(Luuw!|!K77D(+FH " ;-@+ !*~		>f&7CF!&*BDIIN INN(((!..tyy:KL :DDGvDGT!W|-EtyyQDG/tfM/!!	 & 2			F 		&H ** r   c                  ^    t         j                  j                  j                  t               y r-   )r   r?   _custom_op_handlersupdatecustomized_loss_opsr   r   r   r   r   `  s    ..556IJr   c                  l    t         D ]+  } t        j                  j                  j	                  |        - y r-   )r   r   r?   r   pop)	custom_ops    r   r   r   d  s-    ( B	2266yABr   )C
contextlibtypingr   r4   torch._prims_common_prims_commonrP   )torch.distributed._functional_collectivesdistributed_functional_collectivesrX   "torch.distributed.distributed_c10ddistributed_c10drZ   r   torch.distributed.device_meshr   torch.distributed.tensorr   r   r   r	   &torch.distributed.tensor._dtensor_specr
   r   ,torch.distributed.tensor._ops._embedding_opsr   'torch.distributed.tensor._ops._math_opsr   r   r   #torch.distributed.tensor._ops.utilsr   (torch.distributed.tensor.placement_typesr   opsaten__all__contextmanagerr   rE   rq   r)   r:   _ops
OpOverloadobjectdictstrrD   rk   ry   r~   Sizer   r   r   r   default_log_softmax_backward_datanll_loss_forwardnll_loss2d_forwardnll_loss_backwardnll_loss2d_backwardr   r   r   r   r   r   <module>r      s      # : : 1 1  4 G G J E 
 > > yy~~ 
 W W~%	3*? c c (<in-<5?<<6QZZ""Q

Q fQ 	Q06ZZ""

 f 	F'ZZ""'

' f' 	'K K K  TMK  4-	K 
 K  K  K  K  K  K  66>K \\ZZ""\

\ f\ 	\LC5C5C5 C5 TM	C5
 C5 C5 C5 C5 C5 C5 C5 C5LJZZ""J

J fJ 	J\ 	3##++-J!!#<##%>""$>$$&@ KBr   