kh2SdZddlZddlmZmZmZmZddlZddlm Z ddl m Z m Z ddl mZmZmZmZmZmZddlmZddlmZdd lmZdd lmZmZd gZGd d e j<ZGdde j<Z Gdde j<Z!Gdd e j<Z"ddZ#ee#de#de#ddZ$dZ%ddZ&edde"fdZ'edde"fdZ(edde"fdZ)y)a Transformer in Transformer (TNT) in PyTorch A PyTorch implement of TNT as described in 'Transformer in Transformer' - https://arxiv.org/abs/2103.00112 The official mindspore code is released and available at https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT The official pytorch code is released and available at https://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorch N)ListOptionalTupleUnionIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD)MlpDropPath trunc_normal__assert to_2tupleresample_abs_pos_embed)build_model_with_cfg)feature_take_indices) checkpoint)generate_default_cfgsregister_modelTNTc*eZdZdZdfd ZdZxZS) Attentionz Multi-Head Attention ct|||_||_||z}||_|dz|_t j||dz||_t j||||_ t j|d|_ t j|||_ t j|d|_ y)NgbiasT)inplace)super__init__ hidden_dim num_headshead_dimscalennLinearqkvDropout attn_dropproj proj_drop) selfdimr r!qkv_biasr)r+r" __class__s K/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/tnt.pyrzAttention.__init__ s $"*  % ))Cah?3(3It<IIc3' It<c||j\}}}|j|j||d|j|jj ddddd}|j d\}}|j|j|||jdj dddd}||jddz|jz} | jd} |j| } | |zjddj||d}|j|}|j|}|S) Nrrrr-)shaper&reshaper!r"permuteunbindr' transposer#softmaxr)r*r+) r,xBNCr&qkr'attns r0forwardzAttention.forward.s''1a WWQZ  1a G O OPQSTVWYZ\] ^yy|1 FF1I  aDNNB 7 ? ?1a KAKKB''4::5|||#~~d# AX A & . .q!R 8 IIaL NN1 r1)FrG)__name__ __module__ __qualname____doc__rrE __classcell__r/s@r0rrs = r1rc feZdZdZdddddddej ej df fd ZdZxZ S) Blockz TNT Block r4 @FrGct|| ||_t||||| ||_| ||_t |t|dz|| ||_| |_ |jr5| ||_ tj||z|d|_ d|_n=| ||z|_ tj||z|d|_ | ||_| ||_t||||| ||_| dkDr t#| ntj$|_| ||_t |t||z|| ||_y)N)r!r.r)r+r4) in_featureshidden_features out_features act_layerdropTrFrG)rrnorm_inrattn_in norm_mlp_inr intmlp_inlegacy norm1_projr$r%r* norm2_projnorm_outattn_outr Identity drop_pathnorm_mlpmlp)r,r-dim_out num_pixel num_heads_in num_heads_out mlp_ratior.r+r)rcrV norm_layerr]r/s r0rzBlock.__init__BsQ !#   "   &c?aL    ;;(oDO # /7FDI"DO(y9DO # /7GDI(1DO#7+ !  #   1:B),BKKM"7+ ) 34   r1cz||j|j|j|z}||j|j|j |z}|j \}}}|j atj|ddddf|ddddf|j|j|j||dz dzgd}notj|ddddf|ddddf|j |j|j|j||dz dzgd}||j|j|j|z}||j|j|j|z}||fS)Nrrr5r7)rcrYrXr\rZsizer_torchcatr*r^r9rar`rerd)r, pixel_embed patch_embedr?r@rAs r0rEz Block.forwards!DNN4<< [@Y3Z$[[ !DNN4;;t?O?OP[?\3]$^^ ""$1a ?? "))AqsF#AqrE"TYYt{/K/S/STUWX[\W\^`/a%bb%K  ))AqsF#AqrE"T__TYYt{ObObcdfgjkfkmoOp?q5r%ss%K"DNN4=={A[3\$]] !DNN488DMM+s<gg||A F(r1rOceZdZdZ d fd Zd deeeefeffdZdeeefdeeeffdZ de jde jde jfd Z xZ S) PixelEmbedz Image to Pixel Embedding cFt |t|}t|}|d|dz|d|dzf|_|jd|jdz}||_||_||_||_||_|Dcgc]}tj||z } }| |_ tj||jdd||_|j rtj| | |_ytj|||_ycc}w)Nrrr3) kernel_sizepaddingstride)rxrz)rrr grid_sizeimg_size patch_sizer] num_patchesin_dimmathceilnew_patch_sizer$Conv2dr*Unfoldunfold) r,r|r}in_chansrrzr]r~psrr/s r0rzPixelEmbed.__init__s X&z* "1+A6 zRS}8TU~~a(T^^A->?   $ & ;EFR$))BK0FF,IIh AV\] ;;))~VDK)) :NDKGsDreturncH|rt|jS|jSN)maxr})r, as_scalars r0 feat_ratiozPixelEmbed.feat_ratios t' '?? "r1r|cV|d|jdz|d|jdzfS)Nrr)r})r,r|s r0dynamic_feat_sizezPixelEmbed.dynamic_feat_sizes2{dooa00(1+QRAS2SSSr1r> pixel_posc |j\}}}}t||jdk(d|d|d|jdd|jdd t||jdk(d|d|d|jdd|jdd |jrx|j |}|j |}|j ddj||jz|j|jd|jd}nm|j |}|j ddj||jz||jd|jd}|j |}||z}|j||jz|jdj dd}|S) NrzInput image size (*z) doesn't match model (rz).rr5) r8r r|r]r*rr<r9r~rrr})r,r>rr?rAHWs r0rEzPixelEmbed.forwardsWW 1a q! ! 1QC'>t}}Q?O>PPQRVR_R_`aRbQcce f h  q! ! 1QC'>t}}Q?O>PPQRVR_R_`aRbQcce f h ;; ! A AA Aq!))D$$$dkk43F3Fq3I4K^K^_`KacA AA Aq!))!d.>.>*>4??STCUW[WfWfghWijA ! A M IIa$***DKK < F Fq! Lr1)r30r4FT)rHrIrJrKrrrr[rrrnTensorrErLrMs@r0rurus O:#E%S/32F,G# T%S/TeCHoT%,,5<<r1ruc~eZdZdZdddddddd d d d d d d d d d ej d d ffd ZdZejjdZ ejjd'dZ ejjd(dZ ejjdejfdZd)dedeefdZ d*dej(deeeeefdedededededeeej(eej(eej(fffdZ d+deeeefd ed!efd"Zd#Zd'd$efd%Zd&ZxZS),rzC Transformer in Transformer - https://arxiv.org/abs/2103.00112 rrr3tokenirrPr4rQFrGcht||dvsJ||_||_|x|_x|_|_d|_d|_t|||||||_ |jj}t|jdr|jjn|}||_ |jj}|d|dz}|||z|_t!j"||z||_|||_t!j(t+j,dd||_t!j(t+j,d|dz||_t!j(t+j,d||d|d|_t!j4||_t+j8d||Dcgc]}|j;}}g}t=|D]+}|j?tA|||| | | | |||||| -t!jB||_"t=|Dcgc]}tGd ||| c}|_$|||_%t!j4| |_&|dkDrt!j"||nt!jN|_(tS|j.d tS|j0d tS|j2d |jU|jVycc}wcc}w) NravgrF)r|r}rrrzr]rr)p) r-rfrgrhrirjr.r+r)rcrkr]zblocks.)modulenum_chs reduction{Gz?std),rr num_classes global_pool num_featureshead_hidden_size embed_dimnum_prefix_tokensgrad_checkpointingrurpr~hasattrrrr^r$r%r*r_ Parameterrnzeros cls_token patch_posrr(pos_droplinspaceitemrangeappendrO ModuleListblocksdict feature_infonorm head_droprbheadr apply _init_weights)r,r|r}rrrr inner_dimdepthnum_heads_innernum_heads_outerrjr. drop_rate pos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterk first_strider]r~rrrgr>dprrir/s r0rz TNT.__init__s. 2222&&ENNND1DN!""'%!  &&22 -4T5E5E|-TD   ' ' )Zd&))88"1%q(99 $Y%:;IIi)3Y? $Y/ekk!Q &BCekk![1_i&PQekk!Yq@QSabcSd&ef ]3 !&>5!IJAqvvxJJu A MM%!#,-#!((a&%   mmF+ PUV[P\^KLD'! yA F^y) I.9DqBIIi5bkkm dnn#.dnn#.dnn#. 4%%&9K$^s L*L/ct|tjrjt|jdt|tjr8|j +tj j|j dyyyt|tjrUtj j|j dtj j|jdyy)Nrrrg?) isinstancer$r%r weightrinit constant_rs)r,ms r0rzTNT._init_weights2s a # !(( ,!RYY'AFF,>!!!&&!,-?' 2<< ( GG  affa ( GG  ahh ,)r1c hdS)N>rrrr,s r0no_weight_decayzTNT.no_weight_decay;s66r1c$tdddg}|S)Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)i)stemr)r)r,coarsematchers r0 group_matcherzTNT.group_matcher?s!Q)$ r1c||_yr)r)r,enables r0set_grad_checkpointingzTNT.set_grad_checkpointingJs "(r1rc|jSr)rrs r0get_classifierzTNT.get_classifierNs yyr1rrc||_| |dvsJ||_|dkDr&tj|j||_ytj |_y)Nrr)rrr$r%rrbr)r,rrs r0reset_classifierzTNT.reset_classifierRsS&  ""66 66*D >IAoBIIdnnk: SUS^S^S` r1r>indicesreturn_prefix_tokensr stop_early output_fmtintermediates_onlyc |dvsJd|dk(}g} tt|j|\} } |j\} } }}|j ||j }|j |j|j|j| |jd}tj|jj| dd|fd}||jz}|j!|}tj"j%s|s |j}n|jd| dz}t'|D]u\}}|j(r/tj"j%st+|||\}}n |||\}}|| vsR| j-|r|j/|n|w|j0rD| Dcgc]}|ddd|j0f}}| Dcgc]}|dd|j0df} }|ra|jj3||f\}}| Dcgc]6}|j| ||dj5dd dd j78} }tj"j%s|rt9t;| } |r| S|j/|}|| fScc}wcc}wcc}w) a< Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if an int, if is a sequence, select by matching indices return_prefix_tokens: Return both prefix and spatial intermediate tokens norm: Apply norm layer to all intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWNLCz)Output format must be one of NCHW or NLC.rr5rr7Nrr3r)rlenrr8rprr_r*r^r9r~rnrorexpandrrjit is_scripting enumeraterrrrrrr: contiguouslistzip)r,r>rrrrrrr9 intermediates take_indices max_indexr?_heightwidthrprqrrblky prefix_tokensrrs r0forward_intermediateszTNT.forward_intermediatesYs._,Y.YY,& "6s4;;7G"Q i gg1fe&&q$..9 oodii @S@STUW[WgWgik@l0m&no ii!6!6q"b!A; OUVW !DNN2 mmK0 99 ! ! #:[[F[[)a-0F' VFAs&&uyy/E/E/G+5c; +T( [+.{K+H( [L $$tTYY{%;U V  ! !ERSQq!D$:$:"::;SMSDQRqQq$"8"8"99:RMR ##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMlyy%%',@ ]M!BCM  ii , M))!TR msK4K8;K  prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr)rrrr$rbrr)r,rrrrrs r0prune_intermediate_layerszTNT.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r1c j|jd}|j||j}|j|j |j |j ||jd}tj|jj|dd|fd}||jz}|j|}|jD]I}|jr/tj j#st%|||\}}>|||\}}K|j'|}|S)Nrr5rr7)r8rprr_r*r^r9r~rnrorrrrrrrrrr)r,r>r?rprqrs r0forward_featureszTNT.forward_featuress  GGAJ&&q$..9 oodii @S@STUW[WgWgik@l0m&no ii!6!6q"b!A; OUVW !DNN2 mmK0 ;; IC&&uyy/E/E/G+5c; +T( [+.{K+H( [  I ii , r1 pre_logitsc|jr=|jdk(r%|dd|jdfjdn|dddf}|j|}|r|S|j |S)Nrrr7r)rrmeanrr)r,r>rs r0 forward_headzTNT.forward_headsq   =A=M=MQV=V!T++,,-22q29\]^_ab^b\cA NN1 q0DIIaL0r1cJ|j|}|j|}|Sr)rr)r,r>s r0rEz TNT.forwards'  ! !! $   a r1Frr)NFFFrF)rFT)rHrIrJrKr$rsrrrnrignorerrrModulerr[rstrrrrrboolrrrrrrErLrMs@r0rrs ||+R'h- YY77 YY YY)) YY aCahsma8<).$$',G*||G*eCcN34G*#' G*  G*  G*G*!%G* tELL!5tELL7I)I#JJ KG*V./$# 3S >*  $1$1 r1c 8|ddddddttdddd d d |S) Nr)r3rrg?bicubicTzpixel_embed.projrzarXiv:2103.00112zTransformer in TransformerzMhttps://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorch)urlr input_size pool_sizecrop_pct interpolationfixed_input_sizerr first_conv classifier paper_ids paper_name origin_urlr)rkwargss r0_cfgrs;=t'0F('2e    r1ztimm/) hf_hub_id)ztnt_s_legacy_patch16_224.in1kztnt_s_patch16_224.in1kztnt_b_patch16_224.in1kc|jddd|vr|}nxi}|jD]b\}}|jdd}|jdd}|jdd}|jdd }|jd d }|jd d }|jdd}|jdd}|jdd}|jdd}|jdd}|jdd}|jdd}|dk(rh|jjdk(rO|j \}}}t |dzx}} || z|k(sJ|jddd j|||| }|||<e |dj |jj k7r(t|d|jjd !|d<|S)"N outer_tokensr outer_pos inner_posrrqrp proj_norm1r^ proj_norm2r_ inner_norm1rX inner_attnrY inner_norm2rZ inner_mlpr\ outer_norm1r` outer_attnra outer_norm2rd outer_mlpreFg?rrr)new_sizer) popitemsreplacerpr]r8r[r:r9rrr{) state_dictmodelout_dictrCr'r?r@rArrs r0checkpoint_filter_fnr4sNN>4(j $$& DAq +{3A +{3A -7A , 5A , 5A -3A , 2A -7A +x0A -4A , 3A -4A +u-AKE$5$5$<$<$E''1aAH %A1uz!zIIaA&..q!Q:HQK' *U ""eoo&;&;; 6 [ !&&00!  Or1c r|jdd}tt||ftt |dd|}|S)N out_indicesr3getter)r6 feature_cls)pretrained_filter_fn feature_cfg)r.rrr4r)variant pretrainedrr6r2s r0 _create_tntr=sF**]A.K  Wj 1[hG    E Lr1rc Xtddddddd}td d |it|fi|}|S) NrrPFT)r}rrrrr.r]r<)tnt_s_legacy_patch16_224rr=r<r model_cfgr2s r0rBrBsA"at%I  gz gTR[Mf_eMf gE Lr1c Vtdddddd}td d|it|fi|}|S) Nrr?r@rPrAFr}rrrrr.r<)tnt_s_patch16_224rCrDs r0rHrH&s>"aI  ` `d9F_X^F_ `E Lr1c Vtdddddd}td d|it|fi|}|S) Nri(rP FrGr<)tnt_b_patch16_224rCrDs r0rLrL/s>"bI  ` `d9F_X^F_ `E Lr1)rr )*rKrtypingrrrrrntorch.nnr$ timm.datarr timm.layersr r r r rr_builderr _featuresr _manipulater _registryrr__all__r rrOrurr default_cfgsr4r=rBrHrLrr1r0rWs  // E``*+#< ' DY(BIIY(x==@s"))sl %%)&## &  "JCSSr1