kh^ rdZddlZddlZddlZddlmZddlmZm Z m Z m Z ddl Z ddl mcmZddl mZddlmZmZddlmZmZmZmZmZmZddlmZmZmZmZmZd d l m!Z!d d l"m#Z#d d l$m%Z%d d l&m'Z'm(Z(d dl)m*Z*m+Z+m,Z,dgZ-ej\e/Z0GddejbZ2GddejbZ3GddejbZ4de5fdZ6e%de5fdZ7GddejbZ8GddejbZ9d2dejbde:de;fdZd3d"Z?d4d#Z@e+e@e@e@e@d$%e@d$%e@d$%d&ZAe*d3d'e9fd(ZBe*d3d'e9fd)ZCe*d3d'e9fd*ZDe*d3d'e9fd+ZEe*d3d'e9fd,ZFe*d3d'e9fd-ZGe,e/d.d/d0d1y)5a Nested Transformer (NesT) in PyTorch A PyTorch implement of Aggregating Nested Transformers as described in: 'Aggregating Nested Transformers' - https://arxiv.org/abs/2105.12723 The official Jax code is released and available at https://github.com/google-research/nested-transformer. The weights have been converted with convert/convert_nest_flax.py Acknowledgments: * The paper authors for sharing their research, code, and model weights * Ross Wightman's existing code off which I based this Copyright 2021 Alexander Soare N)partial)ListOptionalTupleUnion)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPathcreate_classifier trunc_normal__assert) create_conv2d create_pool2d to_ntupleuse_fused_attn LayerNorm)build_model_with_cfg)feature_take_indices)register_notrace_function)checkpoint_seq named_apply)register_modelgenerate_default_cfgsregister_model_deprecationsNestcdeZdZUdZej j eed<dfd Z dZ xZ S) Attentionz This is much like `.vision_transformer.Attention` but uses *localised* self attention by accepting an input with an extra "image block" dim fused_attncJt|||_||z}|dz|_t |_t j|d|z||_t j||_ t j|||_ t j||_ y)Ng)bias) super__init__ num_headsscalerr#rLinearqkvDropout attn_dropproj proj_drop)selfdimr)qkv_biasr.r0head_dim __class__s L/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/nest.pyr(zAttention.__init__1s ")#% (*99S!C%h7I.IIc3' I.c |j\}}}}|j|j|||d|j||jzj dddddd}|j d\}}} |j r>>QRZZ[\^_abdeghjkl**Q-1a ??..q!QVZVcVc$..BRBRiklADJJAq{{2r**D<>$'DqA IIaAq! $ , ,Q1a 8 IIaL NN1 r7)Fr<r<) __name__ __module__ __qualname____doc__torchjitFinalbool__annotations__r(rS __classcell__r5s@r6r"r"*s' %% /r7r"c`eZdZdZdddddej ej ffd ZdZxZ S)TransformerLayerz This is much like `.vision_transformer.Block` but: - Called TransformerLayer here to allow for "block" as defined in the paper ("non-overlapping image blocks") - Uses modified Attention layer that handles the "block" dimension @Fr<c t || ||_t||||||_|dkDr t |nt j|_| ||_ t||z} t|| |||_ y)N)r)r3r.r0r<) in_featureshidden_features act_layerdrop) r'r(norm1r"rRrrIdentity drop_pathnorm2intr mlp) r1r2r) mlp_ratior3r0r.rjrf norm_layermlp_hidden_dimr5s r6r(zTransformerLayer.__init__\s _     1:B),BKKM_ S9_-*  r7c|j|}||j|j|z}||j|j|j |z}|SN)rhrjrRrmrk)r1rJys r6rSzTransformerLayer.forward{sS JJqM tyy|, , txx 1 67 7r7) rUrVrWrXrGELUrr(rSr^r_s@r6raraVs1gg|| >r7rac&eZdZdfd ZdZxZS)ConvPoolct|t||d|d|_|||_t ddd||_y)Nr%T) kernel_sizepaddingr&maxr:)rxstridery)r'r(rconvnormrpool)r1 in_channels out_channelsropad_typer5s r6r(zConvPool.__init__sB !+|T\cgh |, !%Qq(S r7c0t|jddzdk(dt|jddzdk(d|j|}|j|j ddddj dddd}|j |}|S)z: x is expected to have shape (B, C, H, W) r>r:rz1BlockAggregation requires even input spatial dimsr?r%r)rr@r|r}rBr~r1rJs r6rSzConvPool.forwards  a1$&YZ a1$&YZ IIaL IIaii1a+ , 4 4Q1a @ IIaLr7)rUrVrWr(rSr^r_s@r6rvrvsT r7rv block_sizec|j\}}}}t||zdk(dt||zdk(d||z}||z}|j||||||}|jddj|||zd|}|S)zimage to blocks Args: x (Tensor): with shape (B, H, W, C) block_size (int): edge length of a single square block in units of H, W rz,`block_size` must divide input height evenlyz+`block_size` must divide input width evenlyr:r%r?)r@rrArH)rJrrKHWrN grid_height grid_widths r6blockifyrs ''JAq!Q A Na !OP A Na !NOz/KjJ ![*j*aHA Aq!!![:%=r1EA Hr7c|j\}}}}ttj|}||zx}}|j ||||||}|j ddj ||||}|S)zblocks to image Args: x (Tensor): with shape (B, T, N, C) where T is number of blocks and N is sequence size per block block_size (int): edge length of a single square block in units of desired H, W r:r%)r@rlmathsqrtrArH) rJrrKrL_rN grid_sizeheightwidths r6 deblockifyrstJAq!QDIIaL!I++FU !Y :z1EA Aq!!!VUA6A Hr7c <eZdZdZdddddgdddf fd ZdZxZS) NestLevelz7 Single hierarchical level of a Nested Transformer NrbTr<rct|||_d|_t j t jd||||_|t||| ||_ nt j|_ t| rt| |k(sJdt jt|Dcgc]}t|||| | | | || | c}|_ycc}w)NFr)rorzDMust provide as many drop path rates as there are transformer layers) r2r)rnr3r0r.rjrorf)r'r(rgrad_checkpointingr ParameterrYzeros pos_embedrvr~rilen Sequentialrangeratransformer_encoder)r1 num_blocksr seq_lengthr)depth embed_dimprev_embed_dimrnr3r0r.rjrorfrir5s r6r(zNestLevel.__init__s$ $"'ekk!ZY&WX  % z\deDI DI y>y>U* r,r r*#%==5\ 3# ##!###A,%#  3# $$ 3#s:C"c|j|}|jdddd}t||j}||jz}|j r5t jjst|j|}n|j|}t||j}|jddddS)z+ expects x as (B, C, H, W) rr:r%r) r~rBrrrrrYrZ is_scriptingrrrrs r6rSzNestLevel.forwards IIaL IIaAq ! Q (    " "599+A+A+Ct77;A((+A q$// *yyAq!$$r7)rUrVrWrXr(rSr^r_s@r6rrs0 !,$\%r7rceZdZdZ dfd Zej jddZej jdZ ej jddZ ej jddZ ej jde jfdZdd ed efd Z dd ej$d eeeeefdededededeeej$eej$eej$fffdZ d d eeeefdedefdZdZddefdZdZxZS)!r z Nested Transformer (NesT) A PyTorch impl of : `Aggregating Nested Transformers` - https://arxiv.org/abs/2105.12723 cRt|dD]M}t|}t|tj j s5t||k(rDJd|dt||}t||}t||}||_ |dx|_ |_ g|_ |xst}|xstj}| |_||_t|tj j r|d|dk(sJd|d}||zdk(sJd||_d t'j(|zj+dj-|_||zt1j2|j.dzdk(sJd t5||zt1j2|j.dz|_t9||||dd |_|j:j<|_|j<|j.dz|_g}t'j@d|tC|jE|Dcgc]}|j-}}d }d }tGt|j.D]}||}|jItK|j.||j6|j>||||||| | | | ||||||xjtM||d|gz c_ |}|dz}tjN||_(||d|_)tU|j|j|\}}||_+tjX| |_-||_.|j_|y cc}w)a Args: img_size (int, tuple): input image size in_chans (int): number of input channels patch_size (int): patch size num_levels (int): number of block hierarchies (T_d in the paper) embed_dims (int, tuple): embedding dimensions of each level num_heads (int, tuple): number of attention heads for each level depths (int, tuple): number of transformer layers for each level num_classes (int): number of classes for classification head mlp_ratio (int): ratio of mlp hidden dim to embedding dim for MLP of transformer layers qkv_bias (bool): enable bias for qkv if True drop_rate (float): dropout rate for MLP of transformer layers, MSA final projection layer, and classifier attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate norm_layer: (nn.Module): normalization layer for transformer layers act_layer: (nn.Module): activation layer in MLP of transformer layers pad_type: str: Type of padding to use '' for PyTorch symmetric, 'same' for TF SAME weight_init: (str): weight init scheme global_pool: (str): type of pooling operation to apply to final feature map Notes: - Default values follow NesT-B from the original Jax code. - `embed_dims`, `num_heads`, `depths` should be ints or tuples with length `num_levels`. - For those following the paper, Table A1 may have errors! - https://github.com/google-research/nested-transformer/issues/2  embed_dimsr)depthsz Require `len(z) == num_levels`r?rrz Model only handles square inputsz*`patch_size` must divide `img_size` evenlyr9zUFirst level blocks don't fit evenly. Check `img_size`, `patch_size`, and `num_levels`F)img_size patch_sizein_chansrflattenN)rnr3r0r.rjrorfrzlevels.)num_chs reductionmoduler: pool_type)0r'r(locals isinstance collectionsabcSequencerr num_classes num_featureshead_hidden_size feature_inforrrt drop_rate num_levelsrrYarangefliptolistrrrrlrr patch_embed num_patchesrlinspacesumsplitrappendrdictrlevelsr}r global_poolr- head_drophead init_weights)r1rrrrrr)rrrnr3rproj_drop_rateattn_drop_ratedrop_path_raterorfr weight_initr param_name param_valuerrJdp_ratesprev_dim curr_striderr2rr5s r6r(z Nest.__init__sb ? dJ (:.K+{'?'?@;':5czlRb7cc5 d +Yz*:6 )Ij))4 &:&v.&4>rNBD1,9 ( "$ h  8 8 9A;(1+- Q/Q Q-{H*$)W+WW)$ Z 88>>qAHHJJ&$))DOOA4F*GG1L e d eL x:5$))DOOTUDV:WWX&! m   ++77**dooa.@@(-q.#f+(V(\(\]c(de1AHHJee s4??+, AQ-C MM)"! q #!(("1+%#! "   $skT[\][^R_"`!a a H 1 K+ ,mmV, z"~. .d.?.?AQAQ]hi T&I.  +&IfsN$c|dvsJd|vr tj|j nd}|jD]}t |j dddt tt||y) N)nlhbrrr<{Gz?r>r:stdab) head_bias) rlogrrrrrr_init_nest_weights)r1moderlevels r6rzNest.init_weightssf|###39T>TXXd..//r [[ ?E %//sbA > ?G.)DdKr7cltt|jDchc]}d|d c}Scc}w)Nzlevel.z .pos_embed)rrr)r1rs r6no_weight_decayzNest.no_weight_decays-05c$++6F0GH1&:&HHHs 1c2td|rdnddfddg}|S)Nz ^patch_embedz^levels\.(\d+)z*^levels\.(\d+)\.transformer_encoder\.(\d+))z"^levels\.(\d+)\.(?:pool|pos_embed))r)z^norm)i)stemblocks)r)r1coarsematchers r6 group_matcherzNest.group_matchers0 &,"2_aef=$ r7c4|jD] }||_ yrr)rr)r1enablels r6set_grad_checkpointingzNest.set_grad_checkpointings *A#)A  *r7returnc|jSrr)r)r1s r6get_classifierzNest.get_classifiers yyr7rrcp||_t|j|j|\|_|_y)Nr)rrrrr)r1rrs r6reset_classifierzNest.reset_classifiers2&&7   t//;'H#$)r7rJindicesr} stop_early output_fmtintermediates_onlyc |dvsJdg}tt|j|\}} |j|}t|jdz } t j js|s |j} n|jd| dz} t| D]q\} } | |}| |vs|rL| | k(rG|j|jddddjdddd}|j|a|j|s|r|S | k(r5|j|jddddjdddd}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.rNrr:r%) rrrrrrYrZr enumerater}rBr)r1rJrr}rrr intermediates take_indices max_indexlast_idxstagesfeat_idxstagex_inters r6forward_intermediateszNest.forward_intermediatessU*Y&D(DD& "6s4;;7G"Q i   Q t'!+ 99 ! ! #:[[F[[)a-0F(0 ,OHeaA<'H0"ii !Q1(=>FFq!QPQRG!((1!((+ ,  x  !))Aq!Q/088Aq!DA-r7 prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr)rrrrrir}r)r1rrrrrs r6prune_intermediate_layerszNest.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r7c|j|}|j|}|j|jddddjdddd}|S)Nrr:r%r)rrr}rBrs r6forward_featureszNest.forward_featuressR   Q  KKN IIaii1a+ , 4 4Q1a @r7 pre_logitscp|j|}|j|}|r|S|j|Srr)rrr)r1rJr s r6 forward_headzNest.forward_heads5   Q  NN1 q0DIIaL0r7cJ|j|}|j|}|Srr)r rrs r6rSz Nest.forwards'  ! !! $   a r7)r%r9r%ir9rTr:r:rbTr<r<r<g?NNrravgrF)T)r)NFFrF)rFT)rUrVrWrXr(rYrZignorerrrrrModulerrlstrrTensorrrrr\rrr r rrSr^r_s@r6r r s& )C'J YYLL YYII YY   YY** YY HCHcH8<$$',1 ||1 eCcN341  1  1  1 !%1  tELL!5tELL7I)I#JJ K1 j./$# 3S >*  1$1 r7rnamercVt|tjr|jdrDt |j dddtj j|j|yt |j ddd|j*tj j|jyyt|tjrPt |j ddd|j*tj j|jyyy)zn NesT weight initialization Can replicate Jax implementation. Otherwise follows vision_transformer.py rrr>r:rN) rrr+ startswithrweightinit constant_r&zeros_Conv2d)rrrs r6rrs&"))$ ??6 " &--SB! < GG  fkk9 5 &--SB! <{{&v{{+' FBII &fmma8 ;; " GGNN6;; ' # 'r7ctjd|j|j|jd}|jdd\}}tt j ||z}t |tt j |jdddd}tj|||gdd}t|jddddtt j |}|S) z Rescale the grid of position embeddings when loading from state_dict Expected shape of position embeddings is (1, T, N, C), and considers only square images z$Resized position embedding: %s to %sr:rr%rbicubicF)sizer align_corners) _loggerinfor@rlrrrrBrD interpolater)posemb posemb_newseq_length_oldnum_blocks_newseq_length_newsize_news r6resize_pos_embedr4s  LL7zGWGWX\\!_N%/%5%5a%:"NN499^N:; ? G G1aQR SF ]]68(<9\a bF fnnQ1a0#dii6O2P QF Mr7c|jDcgc]}|jds|}}|D]E}||jt||jk7s*t ||t||||<G|Scc}w)z4 resize positional embeddings of pretrained weights pos_embed_)keysr!r@getattrr4) state_dictmodelrPpos_embed_keyss r6checkpoint_filter_fnr<!s!+!2QAall<6PaQNQ O a=  '%"3"9"9 9,Z]GE1tddddd|}tdd|i|}|S) Nest-B @ 224x224 rrrrrC) nest_baserrErCrD model_kwargsr:s r6rVrVMs<W"jWOUWL  L L| LE Lr7c >tddddd|}tdd|i|}|S) Nest-S @ 224x224 `ir% rrrCrU) nest_smallrWrXs r6rbrbWs3e>ZPZe^deL  M* M ME Lr7c >tddddd|}tdd|i|}|S) Nest-T @ 224x224 r\r_r:r:rTrrCrU) nest_tinyrWrXs r6rfrf`s3d>ZPYd]cdL  L L| LE Lr7c b|jddtddddd|}td d|i|}|S) rTrsamerrrrrCrU) nest_base_jx setdefaultrrErXs r6ririisL j&)W"jWOUWL  OJ O, OE Lr7c b|jddtddddd|}td d|i|}|S) r[rrhr\r_rrrCrU) nest_small_jxrjrXs r6rmrmtsC j&)e>ZPZe^deL  PZ P< PE Lr7c b|jddtddddd|}td d|i|}|S) rdrrhr\r_rerrCrU) nest_tiny_jxrjrXs r6roro~sC j&)d>ZPYd]cdL  OJ O, OE Lr7rirmro) jx_nest_base jx_nest_small jx_nest_tiny)rr<rr)HrXcollections.abcrloggingr functoolsrtypingrrrrrYtorch.nn.functionalr functionalrD timm.datar r timm.layersr r rrrrrrrrr_builderr _featuresr _features_fxr _manipulaterr _registryrrr__all__ getLoggerrUr+rr"rarvrlrrrr rfloatrr4r<rErQ default_cfgsrVrbrfrirmrorUr7r6rs" // A\\ZZ*+34YY ( '  H %) )X)ryy)Xryy(  C    c    ?% ?%DD299DN(ryy((U($" %6 F6"W5#g6"W5& TdTH"$"'r7