^jW,dZddlZddlZddlZddlmZddgZGddej jZGddej jZ Gd d ej jZ Gd dej jZ y) z We will recreate all the RNN modules as we require the modules to be decomposed into its building blocks to be able to observe. N)TensorLSTMCellLSTMc eZdZdZej j ZdgZ dddde de de d dffd Z dd e d e e e fdzd e e e ffd Z dde de d e e e ffdZdZeddZeddZxZS)raA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` `split_gates`: specify True to compute the input/forget/cell/output gates separately to avoid an intermediate tensor which is subsequently chunk'd. This optimization can be beneficial for on-device inference latency. This flag is cascaded down from the parent classes. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(6, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) split_gatesNFr input_dim hidden_dimbiasreturnc||d}t |||_||_||_||_|st jj|d|zfd|i||_ t jj|d|zfd|i||_ t jjjj|_nt jj|_ t jj|_ t jj|_dD]}t jj||fd|i||j|<t jj||fd|i||j|<t jjjj|j|<t jj!|_t jj!|_t jj'|_t jj!|_t jjjj|_t jjjj|_t jjjj|_t jjjj|_d|_d|_t j8|_t j8|_y)Ndevicedtyper )inputforgetcelloutput)?r)super__init__ input_size hidden_sizer rtorchnnLinearigateshgatesao quantizedFloatFunctionalgates ModuleDictSigmoid input_gate forget_gateTanh cell_gate output_gatefgate_cx igate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsquint8hidden_state_dtypecell_state_dtype) selfr r r rrrfactory_kwargsg __class__s n/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/ao/nn/quantizable/modules/rnn.pyrzLSTMCell.__init__,s%+U; #% &+088??1z>,04,8F,DK,188??A N,15,9G,DK+0((++*?*?*O*O*QDJ ((--/DK((--/DK,,.DJ: H!&z"04"8F" A"' "15"9G" A!& 5 5 E E G 1  H ((**, 88++- 88++- --==?  88;;00@@B$)HHKK$9$9$I$I$K! --==? ?G)=E'/4||-2\\xhiddenc| |d|d)|j|jd|j}|\}}|js|j |}|j |}|j j||}|jdd\}} } } |j|}|j| } |j| } |j| } ni} t|j j|jj|j jD]*\\} }}}|j||||| | <,|j| d}|j| d} |j| d} |j| d} |j j#| |}|j$j#|| }|j&j||}|}t)j*|}|j,j#| |}||fS)Nrrrrrr)initialize_hiddenshape is_quantizedrrrr#addchunkr&r'r)r*zipitemsvaluesr+mulr,r-rtanhr.)r4r:r;hxcxrrr#r&r'r)out_gategatekeyr+r,r-cytanh_cyhys r8forwardzLSTMCell.forwardfs  >VAY.&)2C++AGGAJGFB[[^F[[_FJJNN662E;@;;q!;L 8J Y4J**;7Ky1I''1HD03   " ""$ ""$1 =, eff "IIfQi<S  =g7J**4>:KtF|4I''X7H==$$["5&&**:yA #88<zLSTMCell.initialize_hiddens KKT%5%56 7 KKT%5%56 7  "??OWd"==OWd))T9P9PA))T9N9NA!t r9cy)NQuantizableLSTMCellr4s r8 _get_namezLSTMCell._get_names$r9c|du|duk7r td|jd}|jd}||||du|}|stjj ||j _|.tjj ||j _tjj ||j_|.tjj ||j_|St||g||g|j |jgD]\} } } t| jdd| jD])\} } tjj | | _+| at| jdd| jD])\}} tjj || _+|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nz/bi and bh must both be None or both have valuesr=)r r r rrr)dim) AssertionErrorr?rr Parameterrweightr rrCrBrE)clswiwhbibhrrrrwbr#w_chunkrKb_chunks r8 from_paramszLSTMCell.from_paramss $JB$J ' !RS SXXa[ hhqk  "D.#  !&!3!3B!7DKK ~#(88#5#5b#9  !&!3!3B!7DKK ~#(88#5#5b#9    #B8b"X T[[7QR @ 1e%():ELLN%K>MGT"'(("4"4W"=DK>=),QWWQAW-> )O@ $)HH$6$6w$? @  @ r9cxt||jur$td|jdt|t|ds td|j |j |j |j|j|}|j|_ |j|j_ |j|j_ |r`|jjD]}|j|_ |jjD]}|j|_ |S)NExpected module type , got qconfigz$The float module must have 'qconfig'r) type _FLOAT_MODULErehasattrrq weight_ih weight_hhbias_ihbias_hhrurrrE)rhotheruse_precomputed_fake_quantrobservedr6s r8 from_floatzLSTMCell.from_floats  ;c// / '(9(9':&e N ui( !GH H?? OO OO MM MM# # !=="'--"'-- __++- *!MM  *__++- *!MM  *r9TNNN)F)NNF)FF)__name__ __module__ __qualname____doc__rrrrw __constants__intboolrrtuplerPr>rb classmethodrqr __classcell__r7s@r8rrs,HH%%M"OM  8:8:8:8: 8: 8:vAE))!&vv~!6!=) vv~ )X5:-1 vv~ $%##Jr9c teZdZdZ ddddedededdffd Zdd ed eeefdzfd Z e d Z xZ S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. NFrr r r r cV||d}t|t||f||d||_yNr)r r)rrrr) r4r r r rrrr5r7s r8rz_LSTMSingleLayer.__init__s=%+U;  z (,+ IW  r9r:r;cg}|jd}t|D]+}|j|||}|j|d-t j |d}||fS)Nr)r?rangerappendrstack)r4r:r;resultseq_leni result_tensors r8rPz_LSTMSingleLayer.forwardsh''!*w %AYYqtV,F MM&) $ % FA. f$$r9ctj|i|}||j|j|j|j }||_|S)Nr)rrqrrr rr)rhargskwargsrlayers r8rqz_LSTMSingleLayer.from_paramssL##T4V4 OOT--tyydFVFV   r9rr) rrrrrrrrrrPrrqrrs@r8rrs}         %%vv~)>)E%r9rceZdZdZ ddddedededed ed df fd Zdd ed eeefdzfdZ e ddZ xZ S) _LSTMLayerz#A single bi-directional LSTM layer.FNrr r r batch_first bidirectionalr c||d} t |||_||_t ||f||d| |_|jrt ||f||d| |_yyr)rrrrrlayer_fwlayer_bw) r4r r r rrrrrr5r7s r8rz_LSTMLayer.__init__s%+U; &*( z (,+ IW     ,'  ! DM r9r:r;c|jr|jdd}|d\}}n|\}}d}|jr&|d}n |d}|d}|d}n |d}|d}||||f}||d}n>tjj |tjj |f}|j ||\} }t|dr|jr|jd} |j| |\} }| jd} tj| | g| jdz } ||d} d}n|#tjj |\} }n|#tjj |\} }natj|d|dgd} tj|d|dgd}n$| } tjj |\} }|jr| jdd| | |ffS)Nrr=)NNr)r transposerrjit_unwrap_optionalrrxfliprcatrdr transpose_)r4r:r;hx_fwcx_fw hidden_bwhx_bwcx_bw hidden_fw result_fw x_reversed result_bwrrXrYs r8rPz_LSTMLayer.forward0s    Aq!A >'LE5!LE526   }aa}aa U%6!5L =U]I **51 **51I $}}Q : 9 4 $););J#'==Y#G Iy!q)IYY 95y}}7JKF Y%6"33I>A"33I>AKK1y| .>? jj0A0AB zz&%**-jj0A0AB  ?E4G4GH jj6      #   y': Uk)5 6 Uk)5 6 Ui {3T : Ui {3T :)55 B 6    +i[ ABB+i[ ABB)I;h ?FB)I;h ?FB-99BBK:EN r9)TFFNNr)rN) rrrrrrrrrrPrrrrs@r8rrs. !#      855vv~)>)E5n00r9rceZdZdZej j Z ddddededede d e d e d e d e d dffdZ dde de e e fdzfdZdZeddZedZxZS)raXA quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> # xdoctest: +SKIP >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer FNrrr num_layersr rdropoutrrr c  || d t |_|_|_|_|_t|_|_ d_ t|tjr(d|cxkrdkrn tdt|tr td|dkDr9t!j"dd|dk(rt!j"d |d |dt%jjj fdj d  g} | j' fd t)d|Dt*j,j/| _y) NrFrr=zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.) stacklevelzdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=rrrc3K|]>}tjjjfdjd@yw)FrN)rrr r).0_r5r4rs r8 z LSTM.__init__..sY         ""00'  !   sAA)rrrrrr rfloatrrtraining isinstancenumbersNumberr ValueErrorwarningswarnrextendrrr ModuleListlayers)r4rrrr rrrrrrrr5r7s` ` @r8rz LSTM.__init__s%+U; $&$ &W~ * 7GNN3$1$ '4(  Q; MM.   Q BBIK&&0\3 !      ""00'  !      1j)  hh))&1 r9r:r;c|jr|jdd}|jd}|jrdnd}|t j |||j tj|j}|jd|jr#t j|dd|j}t|jDcgc]}||f}}ntjj!|}t#|dt$r|dj'|j|||j } |dj'|j|||j } t|jD cgc]*} | | j)d| | j)df,}} n|}g} g} t+|j,D]s\} }|||| \}\}}| j/tjj!|| j/tjj!|ut j0| }t j0| }|j'd|j2d|j2d}|j'd|j2d|j2d}|jr|jdd}|||ffScc}wcc} w) Nrr=r)rrrrS)rrsizerrrVrrrsqueeze_r@rWrrrrrrrreshapesqueeze enumeraterrrr?)r4r:r;max_batch_sizenum_directionsrVrhxcxhidden_non_optrHrIidxhx_listcx_listrrXrY hx_tensor cx_tensors r8rPz LSTM.forwards    Aq!A"00a >KK  kkxx E NN1 ~~11!''-2$//,BCqUENCDC"YY77?N.+V4#A&..OO^^TEUEU$A&..OO^^TEUEU %T__5W__Q'C);< &#DKK0 :JCac+IAv1 NN59955a8 9 NN59955a8 9 :KK( KK( %%b)//"*=yr?RS %%b)//"*=yr?RS    Aq!A9i(((EDs  K-/K2cy)NQuantizableLSTMr`ras r8rbzLSTM._get_name=s r9c t||js$td|jdt|t |ds |s td||j |j |j|j|j|j|j|}t|d||_ t|jD])}tj!|||d||j"|<+|j$r=|j't(j*j,j/|d }|S|j1t(j*j,j3|d }|S) NrsrtrurrF)rrT)inplace)rrwrervrxrrrr rrrrrurrrrrtrainrr quantization prepare_qatevalprepare)rhr}rurrrs r8rzLSTM.from_float@sZ%!2!23 '(9(9':&e N ui( !VW W          JJ    MM   #  #5)W=))* C#-#8#8sGK$9$HOOC   >> NN xx,,8848PH MMOxx,,44Xt4LHr9ctd)NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rhr}s r8 from_observedzLSTM.from_observedbs" 1  r9)r=TFgFNNr)NF)rrrrrrrrwrrrrrrrPrbrrrrrs@r8rrs6HHMMM !#K2"K2K2K2 K2  K2  K2K2K2K2 K2Z5)5)vv~)>)E5)n!B  r9) rrrrr__all__rModulerrrrr`r9r8rs}  v RuxxRj'uxx'TGGTO 588??O r9