
    ^jZ                         d Z ddlZddlZddlmZmZmZ ddlZddlm	c m
Z ddlmZm	Z	 ddlmZ ddlmZ ddlmZ ded	efd
Z G d de	j*                        Zd!deded	efdZ	 	 	 d"dededz  deeeef      ez  dz  ded	eeef   f
dZ G d de	j*                        Z G d de	j*                        Z G d de	j*                        Zde	j*                  ded	e	j<                  fdZded	efdZ de!d	eegef   fd Z"y)#zTransformer class.    N)CallableOptionalSequence)Tensornn)BuilderArgs)ConditionalQueryInitializer)MSDeformAttndimreturnc                     t        d|       S )z6Clamp a MultiheadAttention head count to at least one.   )maxr   s    d/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/rfdetr/models/transformer.py_safe_multinormalizer      s    q#;    c            
       H     e Zd ZdZdededededdf
 fdZd	edefd
Z xZS )MLPz5Very simple multi-layer perceptron (also called FFN).	input_dim
hidden_dim
output_dim
num_layersr   Nc                     t         |           || _        |g|dz
  z  }t        j                  d t        |g|z   ||gz         D              | _        y )Nr   c              3   N   K   | ]  \  }}t        j                  ||        y wN)r   Linear).0nks      r   	<genexpr>zMLP.__init__.<locals>.<genexpr>*   s     #g1BIIaO#gs   #%)super__init__r   r   
ModuleListziplayers)selfr   r   r   r   h	__class__s         r   r#   zMLP.__init__&   sS    $LJN+mm#gYKRSOUVZdYeUe@f#ggr   xc                     t        | j                        D ]:  \  }}|| j                  dz
  k  rt        j                   ||            n ||      }< |S Nr   )	enumerater&   r   Frelu)r'   r*   ilayers       r   forwardzMLP.forward,   sM    !$++. 	JHAu$%!(;$;uQx qA	Jr   )	__name__
__module____qualname____doc__intr#   r   r2   __classcell__r)   s   @r   r   r   #   sI    ?h# h3 hC hUX h]a h F r   r   
pos_tensorc           	         dt         j                  z  }t        j                  || j                  | j
                        }dd|dz  z  |z  z  }| d d d d df   |z  }| d d d d df   |z  }|d d d d d f   |z  }|d d d d d f   |z  }t        j                  |d d d d dd df   j                         |d d d d dd df   j                         fd      j                  d      }t        j                  |d d d d dd df   j                         |d d d d dd df   j                         fd      j                  d      }| j                  d      dk(  rt        j                  ||fd      }|S | j                  d      d	k(  r| d d d d df   |z  }	|	d d d d d f   |z  }
t        j                  |
d d d d dd df   j                         |
d d d d dd df   j                         fd      j                  d      }
| d d d d df   |z  }|d d d d d f   |z  }t        j                  |d d d d dd df   j                         |d d d d dd df   j                         fd      j                  d      }t        j                  |||
|fd      }|S t        d
j                  | j                  d                  )N   dtypedevicei'  r   r      r      zUnknown pos_tensor shape(-1):{})mathpitorcharanger>   r?   stacksincosflattensizecat
ValueErrorformat)r:   r   scaledim_tx_embedy_embedpos_xpos_yposw_embedpos_wh_embedpos_hs                r   gen_sineembed_for_positionrZ   2   s    KELLJ$4$4Z=N=NOEa5A:&,-EAq!E)GAq!E)GAq$J%'EAq$J%'EKKq!QTTz*..0%1add
2C2G2G2IJPQRZZ[\]EKKq!QTTz*..0%1add
2C2G2G2IJPQRZZ[\]EraiiA. J 
		!Q1W%-1d
#e+U1aA:.224eAq!$Q$J6G6K6K6MNTUV^^_`aQ1W%-1d
#e+U1aA:.224eAq!$Q$J6G6K6K6MNTUV^^_`aiiue4!< J :AA*//RTBUVWWr   memorymemory_padding_maskspatial_shapes	unsigmoidc                    g }d}| j                   \  }}}t        |      D ]W  \  }\  }	}
|h|dd|||	|
z  z   f   j                  ||	|
d      }t        j                  |ddddddf    d      }t        j                  |ddddddf    d      }n`t        j
                  | ddddf   t        j                        |	z   }t        j
                  | ddddf   t        j                        |
z   }t        j                  t        j                  d|	dz
  |	t        j                  | j                        t        j                  d|
dz
  |
t        j                  | j                        d      \  }}t        j                  |j                  d      |j                  d      gd      }t        j                  |j                  d      |j                  d      gd      j                  dddd	      }|j                  d      d
z   |j                         z  }t        j                  |      dz  d|z  z  }t        j                  ||fd      j                  |dd      }|j                  |       ||	|
z  z  }Z t        j                  |d      }|dkD  |dk  z  j!                  dd      }|rdt        j"                  |d|z
  z        }|*|j%                  |j                  d      t        d            }|j%                  | t        d            }nH|*|j%                  |j                  d      t        d            }|j%                  | t        d            }| }|*|j%                  |j                  d      t        d            }|j%                  | t        d            }|j'                  | j(                        |j'                  | j(                        fS )z
    Input:
        - memory: bs, \sum{hw}, d_model
        - memory_padding_mask: bs, \sum{hw}
        - spatial_shapes: nlevel, 2
    Output:
        - output_memory: bs, \sum{hw}, d_model
        - output_proposals: bs, \sum{hw}, 4
    r   Nr   r>   r=   ij)indexingrA   r<   g      ?g?g       @rB   g{Gz?gGz?T)keepdiminf)shaper-   reshaperE   sum
zeros_likelongmeshgridlinspacefloat32r?   rL   	unsqueezefloat	ones_likeappendalllogmasked_filltor>   )r[   r\   r]   r^   	proposals_cur
batch_size_lvlheightwidthmask_flatten_valid_heightvalid_widthgrid_ygrid_xgridrO   proposals_gridwhproposaloutput_proposalsoutput_proposals_validoutput_memorys                           r   gen_encoder_output_proposalsr   N   s    ID||J1 ). 9 _fe*/44&5.;P3Q0QRZZ[egmotvwxM 99mAq!QJ&?%?CL))]1aA:%>$>BK !++F1a7O5::NQWWL**6!Q'?%**MPUUKNN1fqj&fmm\NN1eaiemmFMMZ

 yy&**2.0@0@0DErJ 		;004l6L6LR6PQSTU]]^`bcefhij..+c1U[[]B__^,t3sCx@99nb126>>z2qQ"58 yyA./$6;Kd;RSXXY[eiXj 99%5=M9M%NO*/;;<O<Y<YZ\<]_dej_kl+779O8OQVW\Q]^*/;;<O<Y<YZ\<]_def_gh+779O8OQVWXQYZM&%112E2O2OPR2SUZ[\U]^!--/E.EuQxPMFLL)+;+>+>v||+LLLr   c            3       X    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d)dedededededed	ed
edededededededededededee   dz  dedededededdf2 fdZ	d Z
d Zdedefd Z	 d*d!ee   d"ee   dz  d#ee   d$ed%ed&eeej                   z     dz  deedz  d'f   fd(Z xZS )+TransformerzDTransformer with optional GroupPose keypoint decoder stream support.Nd_modelsa_nheadca_nheadnum_queriesnum_decoder_layersdim_feedforwarddropout
activationnormalize_beforereturn_intermediate_dec
group_detr	two_stagenum_feature_levelsdec_n_pointslite_refpoint_refinedecoder_norm_typebbox_reparamuse_grouppose_keypointsnum_keypoints_per_class grouppose_keypoint_dim_downscalekeypoint_cross_attninter_instance_kp_attnnum_registersdual_projector_kp_onlyr   c                    t         |           d | _        || _        || _        |xs g | _        || _        t        |||||||	|||d||||      }|dv sJ d d d} ||   |      }t        ||||
||||| j
                  |
      | _	        || _
        |rt        j                  t        |      D cg c]  }t        j                  ||       c}      | _        t        j                  t        |      D cg c]  }t        j                   |       c}      | _        |r| j
                  rt%        | j
                        }|dkD  rh||z  }t'        |||      | _        t'        |||      | _        t        j                  t        |      D cg c]  }t-        |||d	       c}      | _        | j1                          |dkD  rjt        j2                  t5        j6                  ||      j9                               | _        t        j2                  t5        j<                  |d
            | _        || _         || _!        || _"        || _#        || _$        || _%        d| _&        y c c}w c c}w c c}w )NF)r   r   r   skip_self_attnenable_keypoint_processingr   r   r   )LNIdentityc                 ,    t        j                  |       S r   )r   	LayerNormchannelss    r   <lambda>z&Transformer.__init__.<locals>.<lambda>   s    2<<#9 r   c                 *    t        j                         S r   )r   r   r   s    r   r   z&Transformer.__init__.<locals>.<lambda>   s     r   )return_intermediater   r   r   r   r   r   r   )out_dimr<   rB   )'r"   r#   encoderr   r   r   r   TransformerDecoderLayerTransformerDecoderdecoderr   r   r$   ranger   
enc_outputr   enc_output_normrg   r	   keypoint_query_initializerkeypoint_query_initializer_encr   enc_out_keypoint_embed_reset_parameters	ParameterrE   emptynormal_register_tokenszerosregister_ref_pointsr   r   
dec_layersr   r   r   _export) r'   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   decoder_layernormdecoder_normrx   total_keypointskeypoint_dimr)   s                                   r   r#   zTransformer.__init__   sa   6 	'>$&<#'>'D"$*/!1% '>-M 3#9
" !$666696
 /t-.w7) 7!5%'>$($@$@-M
 # mmRWXbRc,dQRYYw-H,deDO#%==QVWaQb1cA",,w2G1c#dD &4+G+G"%d&B&B"C"Q&#*.N#NL6Q,7D3 ;V,;D7 35--NST^N_`\7L!D`3D/ 	  1#%<<M70S0[0[0]#^D ')||EKKq4Q'RD$&,$"4(? -e1c as   3I!8I&I+c                     d| _         y NTr   r'   s    r   exportzTransformer.export   	    r   c                     | j                         D ]5  }|j                         dkD  st        j                  j	                  |       7 | j                         D ]#  }t        |t              s|j                          % y r,   )	
parametersr   r   initxavier_uniform_modules
isinstancer
   r   )r'   pms      r   r   zTransformer._reset_parameters  sd    " 	+Auuw{''*	+  	&A!\*##%	&r   maskc                 &   |j                   \  }}}t        j                  |d d d d df    d      }t        j                  |d d dd d f    d      }|j                         |z  }|j                         |z  }t        j                  ||gd      }	|	S )Nr   r   rA   )re   rE   rg   rn   rG   )
r'   r   rx   rz   r{   r}   r~   valid_ratio_hvalid_ratio_wvalid_ratios
             r   get_valid_ratiozTransformer.get_valid_ratio
  s    ::65yy$q!Qw-3iiaAg2$**,v5#))+e3kk=-"@"Er   srcsmasks
pos_embedsrefpoint_embed
query_featcross_attn_srcs.c                    g }|g nd }g }	g }
|g nd }t        t        ||            D ]  \  }\  }}|j                  \  }}}}|
j                  ||f       |j	                  d      j                  dd      }|j	                  d      j                  dd      }|	j                  |       |j                  |       |||   j	                  d      }|j                  |        t        j                  |d      }|It        j                  |d      }t        j                  |D cg c]  }| j                  |       c}d      }t        j                  |	d      }	t        j                  |D cg c]  }t        j                  |      dd  c}      j                  |d   j                  t        j                        }t        j                  |j                  d      |j                  d      j!                  d      d d f      }d }|\g }|D ]?  }t#        |d|      }|j                  |j	                  d      j                  dd             A t        j                  |d      }| j$                  rmt'        |||
| j(                   	      \  }}g g g }}}| j*                  r| j,                  nd} t/        |       D ]  }! | j0                  |!    | j2                  |!   |            }" | j4                  |!   |"      }#| j(                  rm | j6                  |!   |"      }$|$d
d df   |d
dd f   z  |d
d df   z   }%|$d
dd f   j9                         |d
dd f   z  }&t        j:                  |%|&gd      }'n | j6                  |!   |"      |z   }'t=        | j>                  |#j                  d         }(t        j@                  |#jC                  d      d   |(d      d   })t        jD                  |'d|)jG                  d      jI                  ddd            }*|*jK                         }+t        jD                  |"d|)jG                  d      jI                  dd| jL                              },|j                  |+       |j                  |,       |j                  |*        t        j                  |d      }t        j                  |d      }t        j                  |d      }d }-d }.d }/| j$                  rn| jN                  ratQ        | d      rTj                  \  }0}}| jS                  |      }/| j(                  rnjU                         }1| j*                  rtW        | jX                        nd} |/j[                  | d      }2|1j[                  | d      }3g }4t/        |       D ]  }! | jX                  |!   |2|!         }5|3|!   d
dd f   jG                  d      }6|3|!   d
d df   jG                  d      }7|5d
d df   |6z  |7z   }8|4j                  t        j                  |8|5d
dd f   gd              t        j                  |4d      }-|-d
d df   jK                         }.| j\                  dkD  r|j                  d   }9|jG                  d      j_                  |9dd      ja                         }:|jG                  d      j_                  |9dd      ja                         }| j$                  rj                  d   };|d
d |;d d f   }<|d
|;d d d f   }=| j(                  rZ|<d
d df   |d
dd f   z  }>|>|d
d df   z   }>|<d
dd f   j9                         |d
dd f   z  }?t        j:                  |>|?gd      }<n|<|z   }<t        j:                  |<|=gd      }d }@| jb                  dkD  r| j*                  r| j,                  nd}A|:j                  d   |Az  }@| jd                  jG                  d      j_                  |9dd      }B| jf                  jG                  d      j_                  |9dd      }Cti        |:jk                  |@d            }Dti        |jk                  |@d            }Et        j                  |DD Fcg c]  }Ft        j                  |FBgd       c}Fd      }:t        j                  ED Fcg c]  }Ft        j                  |FCgd       c}Fd      }|.|.j                  d   }G| jf                  d d d df   jU                         }H|HjG                  d      jG                  d      j_                  |9d|Gd      }Hti        |.jk                  @d            }It        j                  |ID Fcg c]  }Ft        j                  |FHgd       c}Fd      }.d }J| jN                  r(tQ        | d      stm        d      | jo                  |:      }J| jp                  r||}K|}Ln||n|}Kd }L| js                  |:K||	|||||j                  Kjt                        n|J|.L      }M| jN                  rtW        M      dkD  r
Md d \  }N}O}Pn
Md d \  }N}Od }P| jb                  dkD  r@| j*                  r| j,                  nd}ANj                  d   |Az  }Qt        j                  |Njk                  |Qd      D cg c]  }|d d d d d @d d f    c}d      }Nt        j                  Ojk                  Qd      D cg c]  }|d d d d d @d d f    c}d      }OP\t        j                  Pjk                  Qd      D cg c]  }|d d d d d @f    c}d      }Pn| j$                  sJ d       d }Nd }Od }PNOg}R| j$                  rORj                         | j(                  rRj                         n3Rj                  jU                                nRjw                  d d g       | jN                  rARj                  P       |Rj                  |-       |Rj                  | j$                  r|/nd        ty        R      S c c}w c c}w c c}Fw c c}Fw c c}Fw c c}w c c}w c c}w )Nr<   r   rB   r   )r?   r>   )r   rA   tensors)r^   .r   r   z;use_grouppose_keypoints=True requires keypoint initializers)	memory_key_padding_maskrU   refpoints_unsigmoidlevel_start_indexr]   valid_ratiostgt_keypointsinit_kp_ref_xykp_cross_attn_memoryr@   z,if not using decoder, two_stage must be True)=r-   r%   re   rp   rJ   	transposerE   rL   rG   r   _shape_as_tensorrt   r?   ri   	new_zerosprodcumsumgetattrr   r   r   trainingr   r   r   r   enc_out_class_embedenc_out_bbox_embedexpconcatminr   topkr   gatherrm   repeatdetachr   r   hasattrr   sigmoidlenr   chunkr   expand
contiguousr   r   r   listsplitrM   r   r   r   r>   extendtuple)Sr'   r   r   r   r   r   r   src_flattenmask_flattenlvl_pos_embed_flattenspatial_shapes_hwr   ry   src	pos_embedrx   cr(   wr   r[   r   r]   r   cross_attn_memory
ca_flattentensorr   r   refpoint_embed_ts	memory_tsboxes_tsr   g_idxoutput_memory_gidx!enc_outputs_class_unselected_gidxenc_outputs_coord_delta_gidxenc_outputs_coord_cxcy_gidxenc_outputs_coord_wh_gidx!enc_outputs_coord_unselected_gidxr   topk_proposals_gidxrefpoint_embed_gidx_undetachrefpoint_embed_gidxtgt_undetach_gidxenc_kp_predictionsr   keypoint_memory_tsrw   	boxes_refkp_mem_chunksboxes_chunkskp_pred_chunkskp_deltaref_whref_xykp_xybstgtts_lenrefpoint_embed_ts_subsetrefpoint_embed_subsetrefpoint_embed_cxcyrefpoint_embed_whoriginal_num_queries_per_groupgroup_countreg_tgtreg_ref
tgt_chunks
ref_chunksr   num_keypoints	reg_kp_xykp_ref_chunksr   decoder_memoryr   decoder_outputshs
referenceskeypoint_hs
n_with_regreturn_valuessS                                                                                      r   r2   zTransformer.forward  s    ".rD "35".rD%.s4/D%E 
	*!C!#yJAq!Q$$aV,++a.**1a0C!))!,66q!<I!((3s# Sz))!,##D)
	* ;* 99\15L ;;'OA(<(<Q(?'OQRSL %		*? C RV%W3e&<&<S&A!A&F%WX[[7>> \ 
 "II~'?'?'E~GZGZ[\G]GdGdefGghkikGl&mn !&J& E i5!!&.."3"="=a"CDE !&		*a 8>>.J&7tGXGXCX/+M+ 68R(y,0MMqJz* ">%@T%9%9%%@AWQVAWXeAf%g"4SD4L4LU4STf4g1$$3Q43J3J53QRd3e04S"1"W=@PQTVWVXQX@YY\lmprtstrtmt\uu 0 1MSRSRTW0U0Y0Y0[^nortutvov^w0w-8=46OPVX95
 7//67IJM]] 6 4++-N-T-TUW-XY&+jj1R1V1VWY1Z[\1]_cij&klm&n#/4||5q:M:W:WXZ:[:b:bcdfgij:k0, 'C&I&I&K# %*LL&+>+H+H+L+S+STUWXZ^ZfZf+g%! "(()<=  !23 <=E">H !&		*; C		)3Iyyq1H!!>>d::wtMi?j(J1!%!D!DY!O$($5$58;K;K;MI=A]]T889PQJ.44ZQ4GM$??:1?=LNz* U=466u=mE>RS%e,S!"W5??C%e,S"1"W5??C bqb)F2V;%%eiiab8I0JPR&STU "'>q!A/RaR8??AN??QaB&&q)00R<GGIC+55a8??BKVVXN~~*004+9#ww/+J((6sFGQ(G%$$*B37*KN_`cefeg`gNh*h'*=@QRUWYXYWYRY@Z*Z'(@ab(I(M(M(ORcdgijikdkRl(l%/4||=PRc<djl/m,/GJ[/[,!&/GI^._eg!h .2*!!A%15dooA1411L...88;BB2r2N22<<Q?FFr2rR!#)),JPQ)"RS
!."6"67U[\"6"]^
iiQ[ \E7+;!C \bcd!&\f+gSXEIIug6FA,N+gmn!o!-$2$8$8$;M $ 8 8BQB ? G G II ) 3 3A 6 @ @ C J J2rS`bd eI$()=)=>\bc)=)d$eM%*YYbo/pY^		5):LRS0T/pvw%xN M++t%AB$%bcc $ ? ? D **/@/L!''8$6G6S!2Y_'+$"ll(4)$2"3-FRF^\__^-A-ABdp+-%9 + O ++O0Dq0H.=bq.A+J!0!!4J" !!A%*H*T15dooAXXa[K7
YYJL((S]cd(JefQQq!<<<a?@f #YYJTJZJZ[eklJZJmnQQq!<<<a?@n
 *"'))KVK\K\]gmnK\Kopa1a!@"@!@@Ap#K
 >>Q#QQ>BJKZ(>>  +  $$X.$$X%5%5%78  $.''  -  !34  t~~!34P]##] (P &Xb !]+g 0qT g o
 qs0   $q-q	8 q4 q2 q6q;q"q')i      r@  i,        g        r/   FFr   FrB   rB   Fr   FFNr   TFr   Fr   )r3   r4   r5   r6   r7   rn   strboolr  r#   r   r   r   r   r   r   Moduler  r2   r8   r9   s   @r   r   r      s   N "## !&(-"#%*!%"(-4801$(',',3ii i 	i
 i  i i i i i "&i i i  i i  #!i" #i$ %i& "&'i( "&cT!1)i* +.+i, "-i. !%/i0 1i2 !%3i4 
5iV&F v   @Dk$6lk$ F|d"k$ L	k$
 k$ k$ "&299"45<k$ 
v}c!	"k$r   r   c            "       f    e Zd ZdZ	 	 	 	 	 	 	 	 d$dddedej                  dz  deded	ed
ededee   dz  deddf fdZ	d%dZ
defdZdededefdZ	 	 	 	 	 	 	 	 	 	 	 	 d&dededee   dee   dee   dee   dee   dee   dee   dee   dee   dee   d ee   d!ee   deeed"f   z  fd#Z xZS )'r   z'Decoder stack used by DETR transformer.Nr   r   r   r   r   r   r   r   r   r   r   r   c                    t         |           t        ||      | _        || _        || _        || _        || _        || _        || _	        || _
        |	| _        |
| _        d | _        t        d|z  ||d      | _        d | _        |rN|	rL||
z  }t#        j$                  t'        j(                  t+        |	      |            | _        | j-                          d| _        y )Nr<   F)r"   r#   _get_clonesr&   r   r   r   r   r   r   r   r   r   
bbox_embedr   ref_point_headkeypoint_pos_embedr   r   rE   randnrg   _create_keypoint_class_maskr   )r'   r   r   r   r   r   r   r   r   r   r   kp_dimr)   s               r   r#   zTransformerDecoder.__init__  s     	!-<$	#6 $8!(*D''>$0P- -1!!g+wC"&%*A @@F&(ll5;;sCZ?[]c3d&eD#,,.r   c                     d| _         y r   r   r   s    r   r   zTransformerDecoder.export)  r   r   c                    | j                   s't        j                  ddt        j                        }nt	        | j                         }t        j                  d|z   d|z   t        j                        }d}t        | j                         D ]~  \  }}|dk(  r|t	        | j                   d|       z   }||z   }t        | j                         D ];  \  }}	|	dk(  s||k(  r|t	        | j                   d|       z   }
|
|	z   }d||||
|f<   =  d| j                  v r|| j                  d<   | j                  S | j                  d|d       | j                  S )zDCreate attention mask that blocks cross-class keypoint interactions.r   r`   r   NTkeypoint_class_mask)
persistent)	r   rE   r   rD  rg   r-   _buffersregister_bufferrQ  )r'   r   total_kpoffsetclass_idx_inum_kp_istart_iend_iclass_idx_jnum_kp_jstart_jend_js               r   rM  z.TransformerDecoder._create_keypoint_class_mask,  s_   ++;;q!5::6D4778H;;q8|Q\LDF)243O3O)P 
>%Xq= 3t'C'CL['Q#RR(*-6t7S7S-T >)K1}{(B $s4+G+G+U'VVG#h.E9=D56>
> !DMM137DMM/0 '''   !6 N'''r   r   new_refpoints_deltac                     | j                   rY|dd df   |ddd f   z  |dd df   z   }|ddd f   j                         |ddd f   z  }t        j                  ||gd      }|S ||z   }|S )N.r<   rA   r   )r   r   rE   r   )r'   r   r_  new_refpoints_cxcynew_refpoints_whnew_refpoints_unsigmoids         r   refpoints_refinez#TransformerDecoder.refpoints_refineF  s    #C!G,/B37/KKNabegihigibiNjj   337;??ADWX[]^]_X_D``&+ll4FHX3Y_a&b# '& ':<O&O#&&r   r*  r[   tgt_maskmemory_masktgt_key_padding_maskr   rU   r   r]   r   r   r   r   .c                     |}g }|g}d }d }g } j                   r j                  sJ d       |t        d      |t        d      |} j                  J d        j                  j	                  d      j	                  d      j                  |j                  d   |j                  d   dd      } fd} j                  r5 j                  r ||      \  }}}}n ||j                               \  }}}}t         j                        D ]@  \  }} j                  s5 j                  r ||      \  }}}}n ||j                               \  }}}}d}|z  } j                   r>|< |||f|||||||dk(  |
|	|| j                  |d	}|\  }}|j                  |       n ||||||||||dk(  |
|	
      } j                  sV j                  |      } j                  ||      }| j                  dz
  k7  r|j                  |       |j!                         } j"                  s!|j                   j%                  |             C  j$                  > j%                  |      } j"                  r!|j'                          |j                  |        j"                  r j(                  r1|d   }  j                  |d   }!n|}! j                   r| |!|d   fS | |!fS g }" j                  I|"j                  t+        j,                  |             |"j                  t+        j,                  |             nD|"j                  t+        j,                  |             |"j                  |j	                  d              j                   r$|"j                  t+        j,                  |             t/        |"      S |j	                  d      S )Nz1Keypoint processing requires lite_refpoint_refinezAKeypoint processing is enabled but tgt_keypoints was not providedzBKeypoint processing is enabled but init_kp_ref_xy was not providedz>keypoint_pos_embed must be initialized for keypoint processingr   r   rA   c                 L   | dd df   }j                   r&t        |j                  dz        }|d d d d d f   }nR|d d d d d f   t        j                  gd      d d d f   z  }t        |d d d d dd d f   j                  dz        }j                  |      }||||fS )N.rB   r<   rA   r   )r   rZ   r   rE   rL   rJ  )	refpoints
obj_centerquery_sine_embedrefpoints_input	query_posr'   r   s        r   get_referencez1TransformerDecoder.forward.<locals>.get_reference{  s    "37+J||#=j$,,Z[J[#\ ",Q4Z"8",Q4Z"8599lT`Eace;fghjngn;o"o#=oaQRTUWXj>Y[_[g[gkl[l#m ++,<=I	;KKKr   re  rf  rg  r   rU   rn  rl  is_firstreference_pointsr]   r   keypoint_tgtkeypoint_posrQ  r   )re  rf  rg  r   rU   rn  rl  rq  rr  r]   r   )r   r   rM   rK  rm   r   re   r   r   r-   r&   rQ  rp   rI  rd  r   r   r   r   popr   rE   rG   r  )#r'   r*  r[   re  rf  rg  r   rU   r   r   r]   r   r   r   r   outputintermediatehs_refpoints_unsigmoidrs  kp_query_posintermediate_keypointsro  rk  rm  rn  rl  layer_idr1   pos_transformationlayer_outputsr_  rc  r;  refresultss#   `          `                       r   r2   zTransformerDecoder.forwardQ  s   & "5!6!#**,,a.aa,$ !dee% !eff(L**6x8xx6''11!41**1-|/A/A!/Db"M 	L $$  KXYlKmH
OY8HKXYlYtYtYvKwH
OY8H(5 <	7OHe,,$$O\]pOqLJ<LO\+335PLJ<L
 "#!$66I..<3K %! & +)=,C'%5&!m%4#1&7!-!-(,(@(@)=#!& (5$&--l;% +)=,C'%5&!m%4#1&7  ,,&*oof&=#*.*?*?@SUh*i't22*112IJ&=&D&D&F#''##DIIf$56y<	7| 99 YYv&F''  "##F+##||!"%??.04C-C22s$:2$>>>3wG*u{{<89u{{+ABCu{{<892<<Q?@..u{{+ABC>!""r   )NF   FFFNr   )r   N)NNNNNNNNNNNN)r3   r4   r5   r6   r7   r   rE  rD  r  r#   r   r   rM  rd  r   r  r2   r8   r9   s   @r   r   r     s   1 "&$)%*"+04801#0# # ii$	#
 "# # ## # %)# "&cT!1# +.# 
#J(V (4	'F 	'QW 	'\b 	' &*(,1548 $04.2+/)-*.+/15#\#\# \# 6"	\#
 f%\# 'v.\# "*&!1\# f\# &f-\# $F+\# !(\# v&\#  '\#  !(!\#" 'v.#\#$ 
%$	$%\#r   r   c            (       @    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 d*dededededededed	ed
edededededededdf  fdZde	de
e	   de	fdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d+de	de	de
e	   de
e	   de
e	   de
e	   de
e	   de
e	   de
e	   ded e
e	   d!e	dz  d"e	dz  d#e
e	   d$e
e	   d%e
e	   d&e
e	   de	ee	d'f   z  f$d(Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d+de	de	de
e	   de
e	   de
e	   de
e	   de
e	   de
e	   de
e	   ded e
e	   d!e	dz  d"e	dz  d#e
e	   d$e
e	   d%e
e	   d&e
e	   de	ee	d'f   z  f$d)Z xZS ),r   z9A single decoder layer with optional keypoint subnetwork.r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Nc                    t         |           t        j                  |||d      | _        t        j
                  |      | _        t        j                  |      | _        t        ||	||
      | _
        || _        t        j                  ||      | _        t        j
                  |      | _        t        j                  ||      | _        t        j                  |      | _        t        j                  |      | _        t        j
                  |      | _        t        j
                  |      | _        t)        |      | _        || _        || _        || _        |xr || _        |xr || _        |r||z  }|dkD  rt        j                  ||      nt        j6                         | _        |dkD  rt        j                  ||      nt        j6                         | _        |dkD  rt        j                  ||      nt        j6                         | _        |dkD  rt        j                  ||      nt        j6                         | _        t        j                  |tA        ||z        |d      | _!        t        j
                  |      | _"        t        j                  |      | _#        t        j                  |      | _$        t        j
                  |      | _%        | j2                  r^t        j                  |tA        ||z        |d      | _&        t        j
                  |      | _'        t        j                  |      | _(        | j4                  rTt        ||	tA        ||z        |
      | _)        t        j
                  |      | _*        t        j                  |      | _+        t        j                  ||dz  |z        | _,        t        j
                  |      | _-        t        j                  |dz  |z  |      | _.        t        j
                  |      | _/        t        j                  |      | _0        t        jb                  te        jf                  d      dz        | _4        d| _5        y )NT)	embed_dim	num_headsr   batch_first)n_levelsn_headsn_pointsr   rB   gư>F)6r"   r#   r   MultiheadAttention	self_attnDropoutdropout1r   norm1r
   
cross_attnnheadr   linear1r   linear2norm2norm3dropout2dropout3_get_activation_fnr   r   r   r   r   r   r   inst_in_projinst_pos_in_projinst_out_projmemory_in_projr   kp_inst_self_attnkp_inst_dropoutkp_inst_normkp_norm
kp_dropoutinter_inst_kp_attninter_inst_kp_dropoutinter_inst_kp_normkp_cross_attnkp_cross_attn_dropoutkp_cross_attn_norm
kp_linear1kp_dropout2
kp_linear3kp_dropout4kp_norm5r   rE   onesinstance_kp_layer_scaler   )r'   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rN  r)   s                    r   r#   z TransformerDecoderLayer.__init__  sy   $ 	..H^eswx

7+\\'*
 ''!	
 
 yy/:zz'*yy':\\'*
\\'*


7+

7+,Z8 0$*D'&<&[A[##6#U;U % @@F>^ab>b		'6 :hjhshshuD.NQR.R		'6*XZXcXcXe ! @`bc?c67!;ikititivD@`cd@d"))GV"<jljujujwD%'%:%: .x;[/[\ 	&D" $&::g#6D  "W 5D<</DL jj1DO***,*?*?$28?_3_`# $	+' .0ZZ-@**,,,v*>'''%1/0=]1]^)	&" .0ZZ-@**,,,v*>' ii!?_0_`DO!zz'2D ii!7W(WY_`DO!zz'2DLL0DM+-<<

18L+MD(r   r  rU   c                     ||S ||z   S r    )r'   r  rU   s      r   with_pos_embedz&TransformerDecoderLayer.with_pos_embedX  s    v6&3,6r   r*  r[   re  rf  rg  r   rn  rl  rq  rr  r]   r   rs  rt  rQ  r   .c                    |j                   \  }}}||z   x}}|}| j                  rt        j                  |j	                  || j
                  z  d      d      }t        j                  |j	                  || j
                  z  d      d      }t        j                  |j	                  || j
                  z  d      d      }| j                  |||||d      d   }| j                  r(t        j                  |j	                  |d      d      }|| j                  |      z   }| j                  |      }| j                  | j                  ||      |||||      }|| j                  |      z   }| j                  |      }| j                  | j                  | j                  | j!                  |                        }|| j#                  |      z   }| j%                  |      }| j&                  r||t)        d      | j+                  |      }| j-                  |      }|j                   \  }}}}|j/                  d      }t        j0                  |      j/                  d      }t        j                  ||gd      } t        j                  ||gd      }!| j3                  ||z  d|z   |      } |!j3                  ||z  d|z   |      }!| |!z   x}}| }| j5                  ||||d      d   }"|"j3                  ||d|z   |      }"|"d d d d dd d f   }|"d d d d dd d d f   }#|| j7                  | j9                  |            | j:                  z  z   }| j=                  |      }|| j?                  |#      z   }| jA                  |      }| jB                  r|jE                  dd      j3                  ||z  ||      }$|j/                  d      jG                  ||||      j3                  ||z  ||      }%|$|%z   }|$}| jI                  |||d	      d   }&|&jK                  ||||      jE                  dd      }&|| jM                  |&      z   }| jO                  |      }| jP                  r| j                  ||j/                  d      jG                  ||||            }'|'j3                  |||z  |      }'|j/                  d      jG                  ||||j                   d   |j                   d
         j3                  |||z  |j                   d   |j                   d
         }(||n|})|| jS                  | jU                  |'|(| jW                  |)      |||      j3                  ||||            z   }| jY                  |      }|| j[                  | j]                  | j_                  | j                  | ja                  |                              z   }| jc                  |      }||fS |S )Nr   r   r   F)	attn_maskkey_padding_maskneed_weightszDKeypoint processing is enabled but keypoint_tgt/keypoint_pos missingr<   )r  r  )keyvaluer  r@   )2re   r   rE   rL   r  r   r  r  r  r  r  r  r  r  r   r   r  r  r  r   rM   r  r  rm   rh   rf   r  r  r  r  r  r  r  r   r   r   r  viewr  r  r   r  r  r  r  r  r  r  r  r  )*r'   r*  r[   re  rf  rg  r   rU   rn  rl  rq  rr  r]   r   rs  rt  rQ  r   r)  r   rx   qr    vtgt2
tgt_for_kptgt_for_kp_pos	n_queriesnum_kprN  tgt_expandedquery_expandedcombined_featcombined_poscombined_outkeypoint_tgt2swapped_keypoint_tgtswapped_keypoint_posswapped_outkeypoint_querybbox_ref_for_kp	kp_memorys*                                             r   forward_postz$TransformerDecoderLayer.forward_post[  s   * !YYK
 iA==		!''+"@a'HaPA		!''+"@a'HaPA		!''+"@a'HaPA~~aAL`ot~uvwx==99TZZZ2:DDMM$''jjo Y/#
 DMM$''jjo||DLLc9J)KLMDMM$''jjo**#|'; !ghh**3/J!229=N ,8+=+=(Ay&&%//2L"--j9CCAFN!II|\&BJM 99nl%CKL)11"{2BAJPVWM'//[0@!f*fUL!L00AA11!QEXgl1mnopL'//KVVTL1a
+D(Aqr15M,,T-?-?-EFIeIeeeC##C(C'$//-*HHL<<5L
 **'3'='=a'C'K'KBQWKYdfl'm$",,Q/VBV<WV# % )+??("55aQaV[5\]^_)..r6;OYYZ[]^_+d.H.H.UU#66|D
 ''!%!4!4 .":":1"="D"DRV\^d"e" "0!7!7K&<PRX!Y$..q1V#(..q1(..q1 WRv!57G7M7Ma7PRbRhRhijRkl   5I4T0Z`	+d.H.H&&&'++I6&)/ gb+vv>	/ 	   $66|D
 ($*:*: 0 0Q]A^1_ `a+ L  ==6L $$
r   c                 F    | j                  |||||||||	|
|||||||      S )Nrp  )r  )r'   r*  r[   re  rf  rg  r   rU   rn  rl  rq  rr  r]   r   rs  rt  rQ  r   s                     r   r2   zTransformerDecoderLayer.forward  sO    (   #!5$;--)/%% 3!5# ! 
 	
r   )rB  g?r/   Fr   rB   rB   FFr   TF)NNNNNNNFNNNNNNN)r3   r4   r5   r6   r7   rn   rC  rD  r#   r   r   r  r  r  r2   r8   r9   s   @r   r   r     s   C  $ !&"#$+001$(',!cc c 	c
 c c c c c  c c c %)c +.c "c  !%!c" 
#cJ7V 7(62B 7v 7 &*(,1548 $&*-1-1(,+/)-)-0415'ZZ Z 6"	Z
 f%Z 'v.Z "*&!1Z fZ F#Z #6*Z Z #6*Z Z "D=Z  v&!Z" v&#Z$ &f-%Z& 'v.'Z( 
%$	$)Z@ &*(,1548 $&*-1-1(,+/)-)-0415%&
&
 &
 6"	&

 f%&
 'v.&
 "*&!1&
 f&
 F#&
 #6*&
 &
 #6*&
 &
 "D=&
 v&&
  v&!&
" &f-#&
$ 'v.%&
& 
%$	$'&
r   r   module
num_clonesc                     t        j                  t        |      D cg c]  }t        j                  |        c}      S c c}w r   )r   r$   r   copydeepcopy)r  r  r0   s      r   rH  rH     s-    ==z9JKA$--/KLLKs   A argsc           
      P   t        | dd      }t        di d| j                  d| j                  d| j                  d| j
                  d| j                  d| j                  d	| j                  d
dd| j                  d|d| j                  d| j                  d| j                  d| j                  d| j                  dt        | dd      dt        | dg       dt        | dd      dt        | dd      dt        | dd      dt        | dd      dt        | dd      S )Nr   Fr   r   r   r   r   r   r   r   Tr   r   r   r   r   r   r   r   r   r   r   r   r   num_decoder_registersr   r   r  )r   r   r   	sa_nheads	ca_nheadsr   r   r   r   r   r   r   r   r   r   )r  r   s     r   build_transformerr  $  s~   k51I   $$	
  ,,  ?? !% ??   22 && "66 ++ &&" !(.G O#$ !(.G L%& *17Y[\)]'( $D*?F)*  't-EuM+, d$;Q?-.  't-EuM/ r   r   c                     | dk(  rt         j                  S | dk(  rt         j                  S | dk(  rt         j                  S t	        d|  d      )z-Return an activation function given a string.r/   gelugluz$activation should be relu/gelu, not .)r.   r/   r  r  RuntimeError)r   s    r   r  r  B  sJ    VvvVvvUuu
=j\K
LLr   )   )NNT)#r6   r  rC   typingr   r   r   rE   torch.nn.functionalr   
functionalr.   r   rfdetr.models._typesr   rfdetr.models.heads.keypointsr	   rfdetr.models.ops.modulesr
   r7   r   rE  r   rZ   r  rD  r   r   r   r   r$   rH  r  rC  r  r  r   r   <module>r     sk      / /     , E 2c c 
")) 6  f < *.@D	@M@M$@M U38_-6=@M 	@M
 66>@MFm$")) m$`l# l#^m
bii m
`	M		 Ms Mr}} MK K <M3 M8VHf4D+E Mr   