^j 4dZddlZddlZddlZddlmZddlmZmZm Z m Z m Z m Z m Z ddlZddlmZmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlm Z de de!fdZ"dede e!de e!effdZ#GddeZd1de!de$de$defdZ%dede$defdZ&dede!fdZ'd2de!d e e e!e$fdejPfd!Z)de!de e$fd"Z*d#ede$fd$Z+d#ede,fd%Z-d#ede,fd&Z.d'ede$fd(Z/d)e!d*egefdefd+Z0d,ede!fd-Z1d)e!defd.Z2d/e e!de,fd0Z3y)3z;Xgboost pyspark integration submodule for helper functions.N)Thread)AnyCallableDictOptionalSetTypeUnion)BarrierTaskContext TaskContext) SparkSession)CommunicatorContext)Config)_Args)_ArgVals)Booster)XGBModel) RabitTrackerclsreturnc8|jd|jS)zReturn the class name..) __module____name__)rs ^/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/xgboost/spark/utils.pyget_class_namersnn Qs||n --funcunsupported_setctj|}i}|jjD]C}|j|j us|j |vs+|j||j <E|S)zReturns a dictionary of parameters and their default value of function fn. Only the parameters with a default value will be included. )inspect signature parametersvaluesdefaultemptyname)rr sigfiltered_params_dict parameters r_get_default_params_from_funcr,su   D !C^^**,E    Y__ 4o53<3D3D  0 E rc0eZdZdZdededdffd ZxZS)rz&Context with PySpark specific task ID.contextargsrNc \t|j|d<t| di|y)N dmlc_task_id)str partitionIdsuper__init__)selfr.r/ __class__s rr6zCommunicatorContext.__init__3s+"7#6#6#89^  4 r)rr __qualname____doc__r CollArgsValsr6 __classcell__)r8s@rrr0s&0! 2!L!T!!rrhost n_workersportcd|i}t||d|}|jt|j}d|_|j|j |j |S)z"Start Rabit tracker with n_workersr>task)r>host_ipsortbyr?)targetT)rstartrwait_fordaemonupdate worker_args)r=r>r?r/trackerthreads r_start_trackerrL8s`!9-DYVRVWG MMO 7++ ,FFM LLNKK##%& Krconfc|jJ|jdn |j}t|j||}|S)z3Get rabit context arguments to send to each worker.r)tracker_host_ip tracker_portrL)rMr>r?envs r_get_rabit_argsrRDsE    ++ +!!)1t/@/@D --y$ ?C Jrr.c|jDcgc] }|jjdd"}}|dScc}w)zLGets the hostIP for Spark. This essentially gets the IP of the first worker.:r) getTaskInfosaddresssplit)r.info task_ip_lists r _get_host_iprZLsB;B;O;O;QR4DLL&&s+A.RLR ?Ss%?r(levelctj|}||j|n<|jtjk(r|jtj |j sxtjj sZtjtj}tjd}|j||j||S)zGGets a logger by name, or creates and configures it for the first time.z<%(asctime)s %(levelname)s %(name)s: %(funcName)s %(message)s) logging getLoggersetLevelr[NOTSETINFOhandlers StreamHandlersysstderr Formatter setFormatter addHandler)r(r[loggerhandler formatters r get_loggerrlRs   t $F  <<7>> ) OOGLL ) ??7#4#4#6#?#?'' 3%% J  Y''" Mrctj|}|jtjk(rdS|jS)z+Get the logger level for the given log nameN)r]r^r[r`)r(ris rget_logger_levelrnfs0   t $F<<7>>14Cv||Cr spark_sessionct|rtjS|jjj j |jjj jjdS)z0Gets the current max number of concurrent tasks.r) _is_connectrdmaxsize sparkContext_jscscmaxNumConcurrentTasksresourceProfileManagerresourceProfileFromIdros r_get_max_num_concurrent_tasksrzlsl =!{{  % % * * - - / E E""''**,   !  q ! rc t|tjjjj S#t $rYywxYw)NF) isinstancepysparksqlconnectsessionr AttributeErrorrys rrqrq}s<-)<)<)D)D)Q)QRR s7: AAcv|jjdd}|duxr|dk(xs|jdS)zWhether it is Spark local modez spark.masterNlocalzlocal[)rMget startswith)romasters r _is_localrsA   # #ND 9F   T6W#4#S8I8I(8STr task_contextc| td|j}d|vr tdt|djdj S)z&Get the gpu id from the task resourcesz3_get_gpu_id should not be invoked from driver side.gpuzDCouldn't get the gpu id, Please check the GPU resource configurationr) RuntimeError resourcesint addressesstrip)rrs r _get_gpu_idrs`PQQ&&(I I R   y))!,224 55rmodelxgb_model_creatorcf|}|jt|jd|S)zH Deserialize an xgboost.XGBModel instance from the input model. utf-8) load_model bytearrayencode)rr xgb_models rdeserialize_xgb_modelrs. "#I 5<<#89: rboostercB|jdjdS)z Serialize the input booster to a string. Parameters ---------- booster: an xgboost.core.Booster instance jsonr)save_rawdecode)rs rserialize_boosterrs    F # * *7 33rclt}|jt|jd|S)zN Deserialize an xgboost.core.Booster from the input ser_model_string. r)rrrr)rrs rdeserialize_boosterrs,iG yg!678 Nrdevicec |dvS)z&Whether xgboost is using CUDA workers.)cudarr2)rs ruse_cudars _ $$r)r)N)4r:r"r]rd threadingrtypingrrrrrr r r}r r pyspark.sqlr collectiverCCtxrrCollArgsrr;corersklearnrrJrr3rr,rrLrRrZLoggerrlrnrzboolrqrrrrrrr2rrrsA BBB3$4*1"...   %(X  #s(^ &!$!   C  &SX, S%S/!:gnn(D3D8C=D #"|U\UdU 6k 6c 6 #+BL#9  4w 43 4sw%Xc]%t%r