
    ^j5                     v   d Z ddlmZmZmZmZmZmZmZm	Z	 ddl
ZddlZddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ddlZddlmZ dd	l m!Z! dd
l"m#Z# ddl$mZ% ddl&m'Z' ddlm(Z( ddl)m*Z* ddl+m,Z, ddl+m-Z. ddl/m0Z0 ddl1m2Z2m3Z3 de4ded   deddfdZ5de4ded   deddfdZ6de4ded   deddfdZ7dede4ded   de8ddf
dZ9de8de8de4d e:d!e;ddfd"Z<dede8defd#Z=dedee4   fd$Z>ded%e8d&e8d'e8d(e8de4deej~                  ej                  ej                  f   fd)ZAdede4ddfd*ZBe	d+d+d+d,ded%e8d&e8d-e8d.e;d/ed   d0ej                  j                  deej~                  ej                  f   fd1       Z-e	d+d+d2ded%e8d&e8d-e8d.e;d/e8d0ej                  j                  deej~                  eej                  ej~                  f   f   fd3       Z-d4dej                  d,ded%e8d&e8d-e8d.e;d/e8d0ej                  j                  deej~                  eej                  ej~                  f   f   fd5Z-dede2ddfd6ZEy)7z0Tests for dask shared by different test modules.    )AnyListLiteralTupleTypeUnioncastoverloadN)array)	dataframe)Client
get_worker)parse)make_classification)concat)get_basescore   )dask)EvalsLog)_get_rabit_args)_DASK_VERSION   )make_batches)make_categorical)make_recoded)Deviceassert_allclosetree_methoddevice)cpucudaclientreturnc                    t        ddd      \  }}t        j                  dd| |      }|j                  ||       t	        |      }t        j                  |      j                  d      }t        j                  |      j                  d	      }t        j                  dd| |      }	||	_
        |	j                  ||       t	        |	      }
t        j                  j                  ||
       y
)z%Test init estimation for classsifier.          	n_samples
n_featuresrandom_stater   n_estimators	max_depthr   r   r&   Nchunksr&   N)r   xgbXGBClassifierfitr   da
from_arrayrechunkdxgbDaskXGBClassifierr"   nptestingr   )r   r   r"   Xyclf
base_scoredxdydclfdbase_scores              _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/xgboost/testing/dask.pycheck_init_estimation_clfrF      s     btTDAq


!VC GGAqMs#J	q		!	!	!	4B	q		!	!	!	/B!!	D DKHHR%KJJz;7    c                    ddl m}  |ddd      \  }}t        j                  dd| |      }|j	                  ||       t        |      }t        j                  |      j                  d	
      }t        j                  |      j                  d
      }	t        j                  dd| |      }
||
_        |
j	                  ||	       t        |
      }t        j                  j                  ||       y)z#Test init estimation for regressor.r   )make_regressionr%   r&   r'   r(   r   r,   r/   r0   r2   N)sklearn.datasetsrI   r3   XGBRegressorr5   r   r6   r7   r8   r9   DaskXGBRegressorr"   r;   r<   r   )r   r   r"   rI   r=   r>   regr@   rA   rB   dregrD   s               rE   check_init_estimation_regrO   5   s     1 X"4PDAq


!VC GGAqMs#J	q		!	!	!	4B	q		!	!	!	/B  !VD DKHHR%KJJz;7rG   c                 8    t        | ||       t        | ||       y)zTest init estimation.N)rO   rF   )r   r   r"   s      rE   check_init_estimationrQ   N   s     k66:k66:rG   	n_workersc           	         |dk\  sJ | j                         5  t        j                  ||      }t        j                  t        d      t        ddd      d      }t        j                  g dgdz  dgdz        }t        j                  |j                  d	d
ddf<   | j                  |       |j                  t        j                  ||      t        j                  ||             d	d	d	       y	# 1 sw Y   y	xY w)z0Issue #9271, not every worker has missing value.r   )r   r   i'  r   )abi  r   Ni  i  rU   )rR   npartitions)
as_currentr9   r:   pd	DataFramerangeSeriesr;   nanlocwait_for_workersr5   ddfrom_pandas)r"   r   r   rR   r?   r=   r>   s          rE   check_uneven_nanrc   V   s     >>				 
$$VLLLuU|%q"2EFGII0d
0aS4Z01!#jtDj#o)4NN1)4NN1)4	

 
 
s   CC66C?	worker_id	comm_argsis_qdmc           
          d}d}d}|dk7  }t               j                  j                  }	t        j                  j
                  dddi|5  t        j                  t        |||||       ddi}
|rt        j                  |
|		      }nt        j                  |
|		      }i }t        j                  d
|	|d||dfgd|       t        j                  t        t        t           |d   d               sJ 	 ddd       g g g }}}t!        |      D ]I  }t        |||||      \  }}}|j#                  |       |j#                  |       |j#                  |       K t%        |      }t%        |      }t%        |      }|rt        j&                  ||||	      }nt        j                  ||||	      }i }t        j                  d
|	|d||dfgd|       t(        j*                  j-                  d   d   |d   d   d       y# 1 sw Y   xY w)z-Basic checks for distributed external memory.r&         r    dmlc_communicatorrabit)use_cupyr+   cache)nthreadhist)r   rn   r   Train)evalsnum_boost_roundevals_resultrmseN)weightrn   g-C6?rtol )r   statenthreadsr3   
collectiveCommunicatorContexttmIteratorForTestr   ExtMemQuantileDMatrixDMatrixtrainnon_increasingr	   r   floatr\   extendr   QuantileDMatrixr;   r<   r   )rd   rR   r   re   rf   n_samples_per_batchr*   	n_batchesrl   	n_threadsitXyresultslxlylwixr>   wr=   yconcatwconcatresults_locals                           rE   check_external_memoryr   k   s$    JIH""++I		+	+	Sg	S	S N#!&	
 	
 !77INBR3B		"yFK=/ 	
   d5k773CF3K!LMMM/N2 RBB9 

1a 			!
		!
		!
 	r
ARjGRjG  GGYO[[GGYG MII9G
G}o" JJ -"8"@t  mN Ns   B*HHc                 0    | j                  t        | |      S )z6Get RABIT collective communicator arguments for tests.)syncr   )r"   rR   s     rE   get_rabit_argsr      s    ;;	::rG   c                     t               t        d      k\  rddini } | j                  di |d   }t        |j	                               S )zGet workers from a dask client.z2025.4.0rR   rT   workersrx   )r   parse_versionscheduler_infolistkeys)r"   kwargsr   s      rE   get_client_workersr      sH    "//]:5N"Nk2TVF#f##-f-i8GrG   r)   r*   n_query_groupsmax_relc                   t        |       }|t        |      z  }|dk(  rddlm nddlm dt
        dt
        dt        j                  ffd}g }	d}
t        d||      D ]=  }| j                  |||||
t        |      z     g      }|	j                  |       |
d	z  }
? ||t        |      z  z
  }|dk7  r1| j                  |||t        |      z  
      }|	j                  |        |d	d      }t        j                  |	|      }t        |t        j                        sJ |j                  ddgd	      |j                  |j                  fS )z'Synthetic dataset for learning to rank.r    r   )r[   nseedr#   c                     t         j                  j                  |      }t        | 

d	|      \  }}|j	                  | fd      } |t        
      D cg c]  }d| 	 c}      }||d<   ||d<   |S c c}w )Nr   )n_informativen_redundant	n_classesr+   )sizelowhighf)columnsqidr>   )r;   randomdefault_rngr   integersr\   )r   r   rngr=   r>   r   r   dfDFr   r*   r   s           rE   makezmake_ltr.<locals>.make   s    ii##D)"$
1 ll!.lAU:->?asG?@5	3	 @s   A:
)r   r   r   r   )r   r   metar   r>   )axis)r   lenpandasr[   cudfintrZ   r\   submitappendra   from_delayed
isinstancedropr>   r   )r"   r)   r*   r   r   r   r   n_samples_per_workerr   futuresr   kfutlastr   r   r   s     ```           @rE   make_ltrr      sT    !(G$G4*( 3 2<<    G	A1i!56 mm(q71s7|CS;T:U  
 	s	Q ,s7|;<DqymmDD/Cc'l/RmSs1:D	t	,Bb",,'''77E3<a7("$$66rG   c                    t        | dddd|      \  }}}t        | dddd|      \  }}}t        j                  dd|d	
      }|j                  |||||f||fg||gd       |j                  dk(  sJ |j
                  d   |j                  k(  sJ |j                         d   d   }	t        j                  |	dd d      sJ |	       t        j                  j                  |	d   dd       y)z)Test for the allow_group_split parameter.i      rh      )r   r   r   i   F$   zrank:pairwise)allow_group_splitr-   r   	objectiveT)r   eval_seteval_qidverboser   validation_0zndcg@32Nri   g{Gz?)	tolerancerT   g      ?rv   )r   r9   DaskXGBRankerr5   n_features_in_shapers   r}   non_decreasingr;   r<   r   )
r"   r   X_trq_try_trX_vaq_vay_valtrndcgs
             rE   check_no_group_splitr      s   c!QvD$  c!QvD$ 

!	C GG,t-   $$$::a=C.....n-i8DT#2Y$7==7JJtBx48rG   .)onehot	n_targets	cat_dtypen_categoriesr   r   r   c                     y Nrx   r"   r)   r*   r   r   r   r   s          rE   r   r     s     &)rG   )r   r   c                     y r   rx   r   s          rE   r   r   "  s     ;>rG   Fc                   t        |       }t        |      }g }	|dkD  rt        |      D 
cg c]  }
d|
 	 c}
ndg}dt        dt        j
                  fd} |d||d||      }t        |      D ]T  \  }
}t        ||z  ||
||z  z  z
        }|dk(  r|
d	k(  rd}| j                  ||||||d|g
      }|	j                  |       V t        t        j
                  t	        j                  |	|            }||   }|dk(  r||d	      }||j                  j                  |         }|rt	        j                  |      |fS ||fS c c}
w )z&Synthesize categorical data with dask.r   label_labelr   r#   c                      t        di | \  }}|j                  dk(  r.t        |j                  d         D ]  }|d d |f   |d| <    |S ||d<   |S )Nr   r   r   r   rx   )make_cat_localndimr\   r   )r   r=   r>   r   s       rE   packzmake_categorical.<locals>.packB  sh    ''166Q;1771:& *"#AqD'F1#,*  AgJrG   F)r)   r*   r   r   r   r   r   )r)   r*   r   r   r   r   r   r   )r   r   r\   r   ra   r[   	enumerateminr   r   r	   r   r   
differenceget_dummies)r"   r)   r*   r   r   r   r   r   rR   dfsr   
label_colsr   r   workerl_n_samplesfuturer   r>   r=   s                       rE   r   r   /  s    !(GGI
C 5>MuY/0!6!0y s r||  !D w' 	6"IY)5K0L$L
 >a1fK!!%H  	
 	

6#& BLL"//#D*IJB
:AA~jm
2::  ,-A~~a !##a4Ka 	1s   Ec                     dt         t        j                     dt        dt        dt        j                  fddt         t        j                     ddf fd}t        j                  t        j                  fD ]
  } ||        y)z+Run re-coding test with the Dask interface.DMatrixTargsr   r#   c                     | t         j                  u r|j                  dd       } | |d|i|S |j                  dd         | |i |S )Nref)r9   DaskQuantileDMatrixpop)r   r   r   r   s       rE   create_dmatrixz"run_recode.<locals>.create_dmatrixv  sU     t///**UD)CT5s5f55

5$(((rG   Nc                    t        d      \  }}}}}t              }t        j                  |d      j	                  |      t        j                  |d      j	                  |      t        j                  ||j                  d   dz  f      j	                  |      }}} | ||d	      }	 | ||d|	
      }
t        j                  di|	|
dfg      } | ||d	      }	 | ||d|	
      }
t        j                  di|	|
dfg|d         } | ||d	      }	 | ||d|	
      }
t        j                  di|	|
dfg|d         }t        j                  |d   d   d         j                         sJ t        j                  |d   d   d         j                         sJ t        j                  ||      j                         }t        j                  ||      j                         }t        ||       t        j                   ||	      j                         }t        j                   ||
      j                         }t        ||       y )N`   )r*      rW   )r   r   r0   T)enable_categorical)r  r   r   Valid)rq   booster)rq   	xgb_modelhistoryrt   )r   r   ra   rb   persistr6   r7   r   r9   r   r;   isfiniteallinplace_predictcomputer   predict)r   encreencr>   _todencdreencrB   r   Xy_validr   	results_1	results_2predt_0predt_1r"   r   r   s                   rE   runzrun_recode.<locals>.run  sy   *6bAUAq!' NN3A.66r6BNN5a0888DMM!QWWQZ1_$67???K f HfdB4P!ffbTr
 **Xv&Hg3F2G

 HfdB4P!ffbTr
 JJvg&'i(
	 HffbTR!fdB4R
 JJvg&'i(
	 {{9Y/8@AEEGGG{{9Y/8@AEEGGG&&vw=EEG&&vw?GGI1,,vw3;;=,,vw9AAC1rG   )r   r9   DaskDMatrixr   r   )r"   r   r  r   r   s   ``  @rE   
run_recoder  s  s    )t''()14)@C)			)52d4++, 52 52n %%t'?'?@ HrG   )F__doc__typingr   r   r   r   r   r   r	   r
   numpyr;   r   rZ   r   r   r6   r   ra   distributedr   r   packaging.versionr   r   rJ   r   xgboostr3   xgboost.testingr<   r}   xgboost.compatr   xgboost.testing.updaterr    r9   _typingr   r   
dask.utilsr   datar   r   r   ordinalr   utilsr   r   strrF   rO   rQ   r   rc   dictboolr   r   r   r[   r]   r   r   	DTypeLikeint64r  rx   rG   rE   <module>r1     s   6 I I I      * 4 0   ! 1   " &  4 ! *88%m48>D8	8288%m48>D8	82;;%m4;>D;	;

!$
.5m.D
QT
	
*FFF F 	F
 F 
FR;6 ;c ;c ;
 v  $s)  373737 37
 37 37 37 2<<BII-.37l9 9 9 9> 
 %(	)	)	) 	) 		) 	) qz	) yy""	) 2<<"#	) 
	) 
 %(	>	>	> 	> 		> 	> 	> yy""	> 2<<ryy",,6778	> 
	>$ %'XX@@@ @ 	@ @ @ yy""@ 2<<ryy",,6778@HEv Ev E$ ErG   