khvBdZddlmZddlmZmZmZmZddlZddl m Z ddl m cm Z ddlmZddlmZmZddlmZmZmZmZmZmZmZddlmZmZd d lmZd d lmZd d l m!Z!d d l"m#Z#m$Z$d dl%m&Z&m'Z'dgZ(Gdde jRZ*Gdde jRZ+Gdde jRZ,Gdde jRZ-Gdde jRZ.Gdde jRZ/dedee0e0ffdZ1e!dedee0e0fd e0d!e0fd"Z2Gd#d$e jRZ3Gd%d&e jRZ4Gd'd(e jRZ5Gd)de jRZ6d>d*Z7d+Z8d?d,Z9d@d-Z:e&e:d./e:d./e:d./e:e:e:e:d0dd12e:d3dd12d4Z;e'd?d5e6fd6Ze'd?d5e6fd9Z?e'd?d5e6fd:Z@e'd?d5e6fd;ZAe'd?d5e6fd<ZBe'd?d5e6fd=ZCy)Aaf DaViT: Dual Attention Vision Transformers As described in https://arxiv.org/abs/2204.03645 Input size invariant transformer architecture that combines channel and spacial attention in each block. The attention mechanisms used are linear in complexity. DaViT model defs and weights adapted from https://github.com/dingmyu/davit, original copyright below )partial)ListOptionalTupleUnionN)TensorIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath to_2tuple trunc_normal_Mlp LayerNorm2dget_norm_layeruse_fused_attn)NormMlpClassifierHeadClassifierHead)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpointcheckpoint_seq)generate_default_cfgsregister_modelDaVitc:eZdZddededeffd ZdefdZxZS) ConvPosEncdimkactctt| tj|||d|dz||_|rtj |_ytj|_y)Nr) kernel_sizestridepaddinggroups) superr__init__nnConv2dprojGELUIdentityr")selfr r!r" __class__s M/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/davit.pyr*zConvPosEnc.__init__#sS j$(*II  F   !$2779xcP|j|}||j|z}|SN)r-r")r0r4feats r2forwardzConvPosEnc.forward0s&yy|  r3)F) __name__ __module__ __qualname__intboolr*rr8 __classcell__r1s@r2rr"s) 7C 7C 7$ 7r3rc8eZdZdZdddeffd ZdefdZxZS)Stemz Size-agnostic implementation of 2D image to patch embedding, allowing input size to be adjusted during model forward operation r9`ct|t|}||_||_||_|ddk(sJt j||d|d|_|||_ y)NrrDr9r%r&r') r)r*r r&in_chsout_chsr+r,convnorm)r0rHrIr& norm_layerr1s r2r*z Stem.__init__;sl 6"   ayA~~II     w' r3r4ch|j\}}}}|jd||jdzz |jdz}|jd||jdzz |jdz}tj|d|d|f}|j |}|j |}|S)Nrr)shaper&FpadrJrK)r0r4BCHWpad_rpad_bs r2r8z Stem.forwardQsWW 1aQ!dkk!n"44 AFQ!dkk!n"44 AF EE!a5) * IIaL IIaLr3) r:r;r<__doc__rr*rr8r?r@s@r2rBrB6s' " (,r3rBc0eZdZdeffd ZdefdZxZS) Downsampler9ct|||_||_|||_|dzdk(|_t j|||d|j rdn|dz|_y)Nr$rrG) r)r*rHrIrKeven_kr+r,rJ)r0rHrIr%rLr1s r2r*zDownsample.__init__\sf   v& !Ao* II  #A+*:   r3r4c|j\}}}}|j|}|jrI|jj\}}|||zz |z}|||zz |z} t j |d|d| f}|j|}|S)Nr)rNrKr[rJr%rOrP) r0r4rQrRrSrTk_hk_wrUrVs r2r8zDownsample.forwardqsWW 1a IIaL ;;yy,,HC1s7]c)E1s7]c)Ea!UQ./A IIaLr3)r:r;r<rr*rr8r?r@s@r2rYrY[s "  *  r3rYc&eZdZdfd ZdZxZS)ChannelAttentionV2ct|||_||z|_||_t j ||dz||_t j |||_y)Nr9bias) r)r*r(head_dim dynamic_scaler+Linearqkvr-)r0r num_headsqkv_biasrer1s r2r*zChannelAttentionV2.__init__sS  y( *99S#'9IIc3' r3c:|j\}}}|j|j||d|j||jzj ddddd}|j d\}}}|j r ||dzz}n||jdzz}|jdd|z} | jd } | |jddzjdd}|jddj|||}|j|}|S) Nr9r$rrrDr ) rNrgreshaper(permuteunbindrerd transposesoftmaxr- r0r4rQNrRrgqr!vattns r2r8zChannelAttentionV2.forwards ''1ahhqk!!!Q4;;T[[8HIQQRSUVXY[\^_`**Q-1a   AI ADMMT))A{{2r"Q&|||# AKKB' ' 2 22r : KK1  % %aA . IIaLr3)TT)r:r;r<r*r8r?r@s@r2r`r`}s (r3r`c,eZdZdfd ZdefdZxZS)ChannelAttentionct|||_||z}|dz|_t j ||dz||_t j |||_y)Nrkr9rb)r)r*rhscaler+rfrgr-)r0r rhrirdr1s r2r*zChannelAttention.__init__sU ")#% 99S#'9IIc3' r3r4c |j\}}}|j|j||d|j||jzj ddddd}|j d\}}}||j z}|jdd|z} | jd} | |jddzjdd}|jddj|||}|j|}|S) Nr9r$rrrDrlrmrn) rNrgrorhrprqr}rrrsr-rts r2r8zChannelAttention.forwards''1ahhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1a  N{{2r"Q&|||# AKKB' ' 2 22r : KK1  % %aA . IIaLr3)ryF)r:r;r<r*rr8r?r@s@r2r{r{s(  r3r{cdeZdZdddejej dddffd ZdefdZxZ S) ChannelBlock@FTc t |t|d| |_||_|||_| rt nt} | ||||_|dkDr t|ntj|_ t|d| |_ |jr\|||_t|t!||z||_|dkDrt||_ytj|_yd|_d|_d|_yNr9)r r!r")rhrir) in_featureshidden_features act_layer)r)r*rcpe1ffnnorm1r`r{rxr r+r/ drop_path1cpe2norm2rr=mlp drop_path2) r0r rh mlp_ratiori drop_pathrrLrcpe_actv2 attn_layerr1s r2r*zChannelBlock.__init__s 3!9 _ +-'3C    2;R(9-R[[]3!9 88#CDJ #C)O 4#DH 6?^hy1DODODJDH"DOr3r4c`|j\}}}}|j|jdjdd}|j |}|j |}||j |z}|j|jddj||||}|jw|jdjdd}||j|j|j|z}|jddj||||}|S)Nr$r) rNrflattenrrrrxrrviewrrr)r0r4rQrRrSrTcurs r2r8zChannelBlock.forwardsWW 1a IIaL  # - -a 3jjmiin $ $ IIakk!Q',,Q1a8 9 88  ! &&q!,ADOODHHTZZ]$;<.A;q>S^_`SacdeA !Q1a#..055b!QBA Hr3cjeZdZUdZej j eed<dfd Z de fdZ xZ S)WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True fused_attnc4t|||_||_||_||z}|dz|_t |_tj||dz||_ tj|||_ tjd|_ y)Nrkr9rbrlrn)r)r*r rrhr}rrr+rfrgr-Softmaxrs)r0r rrhrirdr1s r2r*zWindowAttention.__init__s~ &")#% (*99S#'9IIc3' zzb) r3r4c|j\}}}|j|j||d|j||jzj ddddd}|j d\}}}|j rtj|||}n:||jz}||jddz} |j| } | |z}|jddj|||}|j|}|S)Nr9r$rrrDrmrl) rNrgrorhrprqrrOscaled_dot_product_attentionr}rrrsr-) r0r4B_rurRrgrvr!rwrxs r2r8zWindowAttention.forward+s77Aqhhqk!!"aDNNA__annotations__r*rr8r?r@s@r2rrs. %% *r3rcheZdZdZddddej ej ddffd Zdefd Z xZ S) SpatialBlocka< Windows Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm rFrTrFc lt |||_| |_||_t ||_||_t|d| |_ |||_ t||j |||_ |dkDr t|ntj|_t|d| |_|jr^|||_t'||z} t)|| ||_|dkDrt||_ytj|_yd|_d|_d|_yr)r)r*r rrhr rrrrrrrxr r+r/rrrr=rrr) r0r rhrrrirrrLrrmlp_hidden_dimr1s r2r*zSpatialBlock.__init__Ks "$[1"3!9 _ #       2;R(9-R[[]3!9 88#CDJ y1N .#DH 6?^hy1DODODJDH"DOr3r4c 4|j\}}}}|j|jdjdd}|j |}|j ||||}dx}}|j d||j dzz |j dz} |j d||j dzz |j dz} tj|dd|| || f}|j\} } } } t||j }|j d|j d|j dz|}|j|}|j d|j d|j d|}t||j | | }|ddd|d|ddfj}|j |||z|}||j|z}|j|jddj ||||}|jw|jdjdd}||j!|j|j#|z}|jddj ||||}|S)Nr$rrrl)rNrrrrrrrrOrPrrxrrrrrrr)r0r4rQrRrSrTshortcutpad_lpad_trUrV_HpWp x_windows attn_windowss r2r8zSpatialBlock.forwardxsyWW 1a99Q<''*44Q: JJx  FF1aA !!!$q4+;+;A+>'>>$BRBRSTBUU!!!$q4+;+;A+>'>>$BRBRSTBUU EE!aE5%7 8ww 2r1$Q(8(89 NN2t'7'7':T=M=Ma=P'PRST yy+ $((T-=-=a-@$BRBRSTBUWXY <)9)92r B a!RaRlO & & ( FF1a!eQ  tq) ) IIakk!Q',,Q1a8 9 88  ! &&q!,ADOODHHTZZ]$;<s<  gg||+#Z%%r3rceZdZddddddddeej ddd ddffd Zejjdd Z d e fd Z xZ S) DaVitStagerTspatialchannelr9rFr)rrFr$ct|d|_|rt|||| |_nt j |_ g}t|D]}ddlm }g}t|D]a\}}|dk(r)|jdt|||| | || | || f4|dk(s:|jd t|||| | || | || fc|r+|jt j|||jt j|Dcgc]}|d  c}t j||_ycc}w) NF)r%rLr) OrderedDictr spatial_block) r rhrrirrLrrrr channel_block) r rhrrirrLrrrr)r)r*grad_checkpointingrY downsampler+r/range collectionsr enumerateappendrr Sequentialblocks)r0rHrIdepthr attn_typesrhrrridrop_path_ratesrL norm_layer_clrrdown_kernel_size named_blockschannel_attn_v2 stage_blocks block_idxrdual_attention_blockattn_idx attn_typebr1s r2r*zDaVitStage.__init__so( "' (FVcmnDO kkmDO  u ZI /#% '0'< #) )(//,#"+"+!)"1)"<#0 '$/ C 1 )+(//,#"+"+!)"1)"<#0 '* C 1  2##BMM+>R2S$TU##BMMBV3WQAaD3W$XY? Z@mm\2 4Xs Ec||_yr6)r)r0enables r2set_grad_checkpointingz!DaVitStage.set_grad_checkpointings "(r3r4c|j|}|jr6tjj st |j |}|S|j |}|Sr6)rrrr is_scriptingrrr0r4s r2r8zDaVitStage.forwardsS OOA   " "599+A+A+Ct{{A.A AAr3r)r:r;r<rr+rr*rrignorerrr8r?r@s@r2rrsg -"",,!%F3P YY))r3rc.eZdZdZ dfd ZdZejjddZ ejjddZ ejjde jfdZ dded eefd Z dd ej$d eeeeefd ededededeeej$eej$eej$fffdZ dd eeeefdedefdZdZddefdZdZxZS)ra DaViT A PyTorch implementation of `DaViT: Dual Attention Vision Transformers` - https://arxiv.org/abs/2204.03645 Supports arbitrary input sizes and pyramid feature extraction Args: in_chans (int): Number of input image channels. Default: 3 num_classes (int): Number of classes for classification head. Default: 1000 depths (tuple(int)): Number of blocks in each stage. Default: (1, 1, 3, 1) embed_dims (tuple(int)): Patch embedding dimension. Default: (96, 192, 384, 768) num_heads (tuple(int)): Number of attention heads in different layers. Default: (3, 6, 12, 24) window_size (int): Window size. Default: 7 mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True drop_path_rate (float): Stochastic depth rate. Default: 0.1 norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. cft|t|}|t|cxk(rt|k(sJJtt || }tt | | } ||_|dx|_|_||_d|_ g|_ t||d||_ |d}tjd|t|j!|Dcgc]}|j#}}g}t%|D]l}||}t'||f|||dkD| ||||||||| | | |||d}|}|j)||xjt+|d|dzzd| gz c_ nt-j.||_|r@||j |_t5|j |||j |_nBt-j8|_t;|j |||j| |_|j=|j>ycc}w) N)epsrlFr)rL)rrrrhrrrirrLrrrrrrr$zstages.)num_chs reductionmodule) pool_type drop_rate)rrrL) r)r*lenrr num_classes num_featureshead_hidden_sizerr feature_inforBstemrlinspacesumsplittolistrrrdictr+rstagesnorm_prerheadr/rapply _init_weights)r0in_chansdepths embed_dimsrhrrrirLrnorm_epsrrrrrrrdrop_path_rater global_poolhead_norm_first num_stagesrHr4dprrirIstager1s r2r*zDaVit.__init__s60 _ S^:s6{:::::^J7XF } =8L &4>rNBD1""':a=ZH A#(>>!^S[#Q#W#WX^#_`aqxxz``z" cA mGQiq5%#A,'#! #A%+!1 /)#E&F MM%   $w!ac(U\]^\_S`"a!b b / c2mmV, &t'8'89DM&!!%.. DIKKMDM-!!%..% DI 4%%&cas"H.c t|tjrjt|jdt|tjr8|j +tj j|j dyyyy)Ng{Gz?)stdr) isinstancer+rfrweightrcinit constant_)r0ms r2rzDaVit._init_weightsasZ a # !(( ,!RYY'AFF,>!!!&&!,-?' $r3c2td|rdSgdS)Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample)r)z^stages\.(\d+)\.blocks\.(\d+)N)z ^norm_pre)i)rr)r)r0coarses r2 group_matcherzDaVit.group_matchergs'(.$  5  r3cX||_|jD]}|j|y)N)r)rrr)r0rrs r2rzDaVit.set_grad_checkpointingrs."([[ 8E  ( ( ( 7 8r3returnc.|jjSr6)rfc)r0s r2get_classifierzDaVit.get_classifierxsyy||r3rrcJ||_|jj||yr6)rrreset)r0rrs r2reset_classifierzDaVit.reset_classifier|s&  [1r3r4indicesrK stop_early output_fmtintermediates_onlycr|dvsJdg}tt|j|\}} |j|}t|jdz } tj j s|s |j} n|jd| dz} t| D]u\} } |jr+tj j s t| |}n| |}| |vsJ|r| | k(r|j|}n|}|j|w|r|S | k(r|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.rN) rrrrrrrrrrrr)r0r4rrKrrr intermediates take_indices max_indexlast_idxrfeat_idxrx_inters r2forward_intermediateszDaVit.forward_intermediatess,*Y&D(DD& "6s4;;7G"Q i IIaLt{{#a' 99 ! ! #:[[F[[)a-0F(0 .OHe&&uyy/E/E/Gua(!H<'H0"mmA.GG$$W- .  x  a A-r3 prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrr)rrrr+r/rr)r0rr$r%rrs r2prune_intermediate_layerszDaVit.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1 KKMDM   ! !!R (r3c|j|}|jr5tjj st |j |}n|j |}|j|}|Sr6)rrrrrrrrrs r2forward_featureszDaVit.forward_featuressW IIaL  " "599+A+A+Ct{{A.A AA MM! r3 pre_logitscN|r|j|dS|j|S)NT)r+)r)r0r4r+s r2 forward_headzDaVit.forward_heads$0:tyyty,L ! Lr3cJ|j|}|j|}|Sr6)r*r-rs r2r8z DaVit.forwards'  ! !! $   a r3)r9rrr9rrCr9 rFrDT layernorm2d layernormgh㈵>rTFr$FFrravgFFrr6)NFFrF)rFT)r:r;r<rWr*rrrrrrr+Modulerr=rstrrrrrr>rr#r(r*r-r8r?r@s@r2rrs&*$$%-!!-W'r-  YY   YY88  YY 2C2hsm28<$$',3 ||3 eCcN343  3  3  3 !%3  tELL!5tELL7I)I#JJ K3 n./$# 3S >*  M$Mr3cNddl}i}|jD] \}}|j|r|j|d}n,|j dd|}|j dd|}|jdd}|jd d }|jd d }|jd d}|jdd }|jdd}|jdd}|jdd}|jdd}|jdd}|||< |S)Nrr'zconvs.([0-9]+)stages.\1.downsamplezblocks.([0-9]+)stages.\1.blocksdownsample.projdownsample.convstages.0.downsamplerzwindow_attn.norm.znorm1.zwindow_attn.fn.zattn.zchannel_attn.norm.zchannel_attn.fn.z ffn.norm.znorm2.z ffn.fn.net.zmlp.z conv1.fn.dwz cpe1.projz conv2.fn.dwz cpe2.proj)reitems startswithreplacesub) state_dictmodelprefixrEout_dictr!rws r2_convert_florence2rNs& H  "1 <<  &"%A  FF$&=q A FF%':A > II'): ; II+V 4 II)8 4 II' 1 II*H 5 II(' 2 IIk8 , IImV , IIm[ 1 IIm[ 1 '* Or3cd|vr|Sd|vr|d}d|vr t||Sddl}i}|jD]\}}|jdd|}|jdd |}|j d d }|j d d }|j dd}|j dd}|j dd}|j dd}|||<|S)z Remap MSFT checkpoints -> timm zhead.fc.weightrJz vision_tower.convs.0.proj.weightrNzpatch_embeds.([0-9]+)r@zmain_blocks.([0-9]+)rArBrCrDrzhead.zhead.fc.znorms.z head.norm.zcpe.0rzcpe.1r)rNrErFrIrH)rJrKrErMr!rws r2checkpoint_filter_fnrPs:%z! - )Z7!*e44 H  " 1 FF+-Da H FF*,? C II'): ; II+V 4 IIgz * IIh - IIgv & IIgv &   Or3c tdt|jddD}|jd|}|jdd}|j drd}t t ||fttd| |d |}|S) Nc3&K|] \}}| ywr6).0rrs r2 z _create_davit..s\da\srr/ out_indicespretrained_strictT_flF)flatten_sequentialrV)pretrained_filter_fn feature_cfgrW) tuplergetpopendswithrrrPr)variant pretrainedkwargsdefault_out_indicesrVstrictrKs r2 _create_davitre s\i 8\8Z.[\\**],?@K ZZ+T 2F   2DkJ    E Lr3c 0|dddddttddd |S) Nr:)r9rg)rFrFgffffff?bicubicz stem.convzhead.fc) urlr input_size pool_sizecrop_pct interpolationmeanr first_conv classifierr )rirbs r2_cfgrq"s0=v9%.B!     r3ztimm/) hf_hub_idzmicrosoft/Florence-2-base)r9r3r3)rrrrjzmicrosoft/Florence-2-large)zdavit_tiny.msft_in1kzdavit_small.msft_in1kzdavit_base.msft_in1k davit_large davit_huge davit_giantzdavit_base_fl.msft_florence2zdavit_huge_fl.msft_florence2rc Ltddd}tdd|it|fi|S)Nr/r0r4rrrhra) davit_tinyrrerarb model_argss r2rxrxBs0\6IUcdJ  [* [Z@ZSY@Z [[r3c Ltddd}tdd|it|fi|S)Nrr rr0r4rwra) davit_smallryrzs r2rrHs0\6IUcdJ  \: \jA[TZA[ \\r3c Ltddd}tdd|it|fi|S)Nr}rDry rwra) davit_baseryrzs r2rrNs0\6KWefJ  [* [Z@ZSY@Z [[r3c Ltddd}tdd|it|fi|S)Nr})r1r2r3)r5r6r70rwra)rsryrzs r2rsrsTs0\6KWfgJ  \: \jA[TZA[ \\r3c Ltddd}tdd|it|fi|S)Nr}rrriryrr@rwra)rtryrzs r2rtrtZs0\6LXghJ  [* [Z@ZSY@Z [[r3c Ltddd}tdd|it|fi|S)N)rrr6r9)r2r3ri )r6r7rrCrwra)ruryrzs r2ruru`s0]7MYijJ  \: \jA[TZA[ \\r3c Ttddddddd}td d|it|fi|S) Nr}rrr6r9Trrrhrrrrra) davit_base_flryrzs r2rrgs=(=DtJ  ^Z ^4 C]V\C] ^^r3c Ttddddddd}td d|it|fi|S) Nr}rrr6r9Trra) davit_huge_flryrzs r2rrps?(>/DtJ  ^Z ^4 C]V\C] ^^r3)z vision_tower.r<)r')DrW functoolsrtypingrrrrrtorch.nnr+torch.nn.functional functionalrOr timm.datar r timm.layersr r rrrrrrr_builderr _featuresr _features_fxr _manipulaterr _registryrr__all__r=rrBrYr`r{rr=rrrrrrrNrPrerq default_cfgsrxrrrsrtrurrrSr3r2rs // Alll=*+33< )("299"JD>ryy4:299:z  U38_   F  sCx  S  S    )bii)X_299_DSSl^BII^B84*% ! 6&6$(--%1%).-%1& (\e\\ ]u]] \e\\ ]u]] \e\\ ]u]] ______r3