^j)vdZddlmZddlmZmZmZmZddlm Z ddl m Z ddl m Z Gdd e jZGd d e jZeee j"d ZGdde jZGdde jZGdde jZGdde jZy)zl MLP module w/ dropout and configurable activation layer Hacked together by / Copyright 2020 Ross Wightman )partial)OptionalTypeUnionTuple)nn)GlobalResponseNorm) to_2tupleceZdZdZddej ddddddf dedeedeed eejd eeejd e e e e e ffd e e e e e ffd e ffd ZdZxZS)Mlpz MLP as used in Vision Transformer, MLP-Mixer and related networks NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected. NTF in_featureshidden_features out_features act_layer norm_layerbiasdropuse_convc | | d} t||xs|}|xs|}t|}t|} |rttj dntj } | ||fd|di| |_||_t j| d|_ | ||fi| nt j|_ | ||fd|di| |_ t j| d|_y)Ndevicedtyper  kernel_sizerr)super__init__r rrConv2dLinearfc1actDropoutdrop1Identitynormfc2drop2)selfrrrrrrrrrrdd drop_probs linear_layer __class__s Z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/mlp.pyrz Mlp.__init__s / #2{ )8[t_ H>TJ!3:r:Z\ZeZeZg 1 4lWaWTVWZZ 1 . r/c|jjZtjj |jj|jjj ddzdtjj |jj|jjj ddzddy)NrrEư>std)r!rrinitones_shapenormal_weightr)s r. init_weightszGluMlp.init_weights_s 88== $ GGMM$((-- (;(;A(>!(C(DE F (=(=a(@A(E(FGTRr/cX|j|}|jd|j\}}|jr||j |zn|j ||z}|j |}|j |}|j|}|j|}|S)NrE)dim) r!chunkrGrCr"r$r&r'r()r)r3x1x2s r.r4zGluMlp.forwardes HHQK/B!%B" TXXb\B5F JJqM IIaL HHQK JJqMr/)r6r7r8r9rSigmoidr;rrr<rr=rr>rrRr4r?r@s@r.rBrB9s.2*.)+483768""//&c]/#3- / BII / !bii1 /eD$J//0/uUE\223///@S r/rBF)rrCceZdZdZddej ddddddf dedeedeed eejd eeejd e e e e e ffd e e e e e ffd effd ZdZdZxZS)SwiGLUz SwiGLU NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and better matches some other common impl which makes mapping checkpoints simpler. NTrrrrrrrrralign_toc  | | d} t ||xs|}|xs|}t|}t|} |r || |zz}tj||fd|di| |_tj||fd|di| |_||_tj| d|_ | ||fi| ntj|_ tj||fd|di| |_ tj| d|_ y)Nrrrr )rrr rr fc1_gfc1_xr"r#r$r%r&r'r()r)rrrrrrrr[rrr*r+r-s r.rzSwiGLU.__init__xs / #2{ )8[t_ -/1AH1LMOYY{OP$q'PRP YY{OP$q'PRP ;ZZ 1 . 9C9OJ5"5UWU`U`Ub 99_lOaOBOZZ 1 . r/c|jj3tjj |jjtjj |jj dy)NrIrJ)r]rrrLrMrOrPrQs r.rRzSwiGLU.init_weightssF ::?? & GGMM$**// *  ))t4r/c|j|}|j|}|j||z}|j|}|j |}|j |}|j |}|Sr1)r]r^r"r$r&r'r()r)r3x_gates r.r4zSwiGLU.forwardshA JJqM HHV q  JJqM IIaL HHQK JJqMr/)r6r7r8r9rSiLUr;rrr<rr=rr>rrRr4r?r@s@r.rZrZss.2*.)+483768//&c]/#3- / BII / !bii1 /eD$J//0/uUE\223//>5 r/rZceZdZdZddej ddddddf dedeedeedeejd eeejd eeejd e e e e e ffd e e e e e ffffd ZdZxZS)GatedMlpz MLP as used in gMLP NTrrrrrr gate_layerrrc :| | d} t ||xs|}|xs|}t|}t|} tj||fd|di| |_||_tj| d|_||dzdk(sJ||fi| |_ |dz}ntj|_ | ||fi| ntj|_ tj||fd|di| |_ tj| d|_ y)NrrrrEr )rrr rr r!r"r#r$gater%r&r'r()r)rrrrrrerrrrr*r+r-s r.rzGatedMlp.__init__s / #2{ )8[t_ 99[/NQN2N;ZZ 1 .  !"Q&!+ ++"?9b9DI-2O DI9C9OJ5"5UWU`U`Ub 99_lOaOBOZZ 1 . r/c|j|}|j|}|j|}|j|}|j |}|j |}|j |}|Sr1)r!r"r$rgr&r'r(r2s r.r4zGatedMlp.forwardsb HHQK HHQK JJqM IIaL IIaL HHQK JJqMr/r5r@s@r.rdrds .2*.)+48483768//&c]/#3- / BII / !bii1 /!bii1/eD$J//0/uUE\223/Br/rdceZdZdZddej dddddfdedeedeedeejd eeejd e e e e e ffd e ffd Zd ZxZS)ConvMlpzG MLP using 1x1 convs that keeps spatial dims (for 2D NCHW tensors) NTrrrrrrrrc v|| d} t ||xs|}|xs|}t|}tj||fd|dd| |_|r ||fi| ntj |_||_tj||_ tj||fd|dd| |_ y)Nrr r)rr) rrr rrr!r%r&r"r#rr') r)rrrrrrrrrr*r-s r.rzConvMlp.__init__s / #2{ )8[99[/]qtTUw]Z\]9CJ5"5 ;JJt$ 99_l^PTUVPW^[]^r/c|j|}|j|}|j|}|j|}|j |}|Sr1)r!r&r"rr'r2s r.r4zConvMlp.forwardsH HHQK IIaL HHQK IIaL HHQKr/)r6r7r8r9rReLUr;rrr<rr=rr>rr4r?r@s@r.rjrjs .2*.)+4837__&c]_#3- _ BII _ !bii1 _eD$J//0__0r/rjceZdZdZddej dddddfdedeedeed eejd e e e e e ffd e e e e e ffd e ffd ZdZxZS)GlobalResponseNormMlpz MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d NOTE: Intended for '2D' NCHW (use_conv=True) or NHWC (use_conv=False, channels-last) tensor layouts NTrFrrrrrrrc || d} t ||xs|}|xs|}t|}t|} |rttj dntj } | ||fd|di| |_||_t j| d|_ t|fd| i| |_ | ||fd|di| |_ t j| d|_y)Nrr rrr channels_last)rrr rrrr r!r"r#r$r grnr'r()r)rrrrrrrrrr*r+r,r-s r.rzGlobalResponseNormMlp.__init__s / #2{ )8[t_ rzs//#("))(V4RYY4nvEB 2RYY2j,ryy,^!bii!H'BII'r/