khvdZddlZddlmZddlmZmZmZmZddl Z ddl m Z ddl m Z mZddlmZmZmZmZmZddlmZdd lmZdd lmZdd lmZdd lmZmZm Z dd l!m"Z"dgZ#eGdde jHZ%ddZ&Gdde jHZ'Gdde jHZ(Gdde jHZ)Gdde jHZ*Gdde jHZ+Gdde jHZ,dZ-ddZ.ddZ/eid e/d!d"#d$e/d!d%#d&e/d!d'd()d*e/d!d+#d,e/d!d-#d.e/d!d/d()d0e/d!d1#d2e/d!d3#d4e/d!d5d()d6e/d!d7#d8e/d!d9#d:e/d!d;d()d<e/d!d=#d>e/d!d?#d@e/d!dAd()dBe/d!dC#dDe/d!dE#idFe/d!dGd()dHe/d!dI#dJe/d!dK#dLe/d!dMd()dNe/d!dO#dPe/d!dQ#dRe/d!dSd()dTe/d!dU#dVe/d!dW#dXe/d!dYd()dZe/d!d[#d\e/d!d]#d^e/d!d_d()d`e/d!da#dbe/d!dc#dde/d!ded()dfe/d!dg#e/d!dh#e/d!did()e/d!dj#e/d!dk#e/d!dld()e/d!dm#e/d!dn#e/d!dod()dpZ0eddqe,fdrZ1eddqe,fdsZ2eddqe,fdtZ3eddqe,fduZ4eddqe,fdvZ5eddqe,fdwZ6eddqe,fdxZ7eddqe,fdyZ8eddqe,fdzZ9eddqe,fd{Z:eddqe,fd|Z;eddqe,fd}Zeddqe,fdZ?eddqe,fdZ@eddqe,fdZAeddqe,fdZBeddqe,fdZCeddqe,fdZDeddqe,fdZEeddqe,fdZFeddqe,fdZGeddqe,fdZHeddqe,fdZIeddqe,fdZJeddqe,fdZKeddqe,fdZLe eMidd$dd&dd,dd.dd2dd4dd8dd:dd>dd@ddDddFddJddLddPddRddVdXd\d^dbddddddddd y)a] Cross-Covariance Image Transformer (XCiT) in PyTorch Paper: - https://arxiv.org/abs/2106.09681 Same as the official implementation, with some minor adaptations, original copyright below - https://github.com/facebookresearch/xcit/blob/master/xcit.py Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman N)partial)ListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath trunc_normal_ to_2tupleuse_fused_attnMlp)build_model_with_cfg)feature_take_indices)register_notrace_module) checkpoint)register_modelgenerate_default_cfgsregister_model_deprecations) ClassAttnXcitc8eZdZdZdfd ZdededefdZxZS)PositionalEncodingFourierz Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper. Based on the official XCiT code - https://github.com/facebookresearch/xcit/blob/master/xcit.py ct|tj|dz|d|_dt j z|_||_||_ ||_ d|_ y)Nr) kernel_sizeư>) super__init__nnConv2dtoken_projectionmathpiscale temperature hidden_dimdimeps)selfr)r*r( __class__s L/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/xcit.pyr!z"PositionalEncodingFourier.__init__)sS  " *q.#1 M[ &$BHWc |jjj}|jjj}t j d|dz|j tjjdjdd|}t j d|dz|j tjjd|d}||ddddddf|jzz |jz}||ddddddf|jzz |jz}t j |j|j tj}|jdt j|ddz|jz z}|dddddddf|z } |dddddddf|z } t j| dddddddddfj!| dddddddddfj#gd j%d } t j| dddddddddfj!| dddddddddfj#gd j%d } t j&| | fd j)dd dd} |j| j |} | j|dddS) Nr)devicerfloor) rounding_moderr*)r$weightr4dtypetorcharangetofloat32 unsqueezerepeatr+r'r)r(divstacksincosflattencatpermute) r,r0r1r2r4r<y_embedx_embeddim_tpos_xpos_yposs r.forwardz!PositionalEncodingFourier.forward2s&&--44%%,,22,,q!a%7::5==ISSTUV]]^_abdef,,q!a%7::5==IPPQRTUWXYWQQY/$((:;djjHWQ23Y/$((:;djjH T__V<?? N  Q5!7)S%SVZVeVe%ef1a &.1a &. U1aADqD=1557q!Q1}9M9Q9Q9STZ[\ddefg U1aADqD=1557q!Q1}9M9Q9Q9STZ[\ddefgiiA.66q!QB##CFF5M2zz!Q1%%r/) i')__name__ __module__ __qualname____doc__r!intrP __classcell__r-s@r.rr!s' &&&&r/rc tjjtj||d|ddtj|S)z3x3 convolution + batch normr:rF)rstridepaddingbias)r=r" Sequentialr# BatchNorm2d) in_planes out_planesr[s r.conv3x3rbDs< 88   )ZQvqW\] z" r/cHeZdZdZddddej ffd ZdZxZS)ConvPatchEmbedz|j|}tj|j||ddddfgd}||j |j |zz}|j r|j|}n:tj|j|ddddf|ddddfgd}|}|ddddf}|j|j|z}tj||ddddfgd}||j |z}|S)Nrr9r) rr=rHrrrrrrr)r,rsx_norm1x_attnx_res cls_tokens r.rPzClassAttentionBlock.forwards**Q-DIIg.12?QG t{{V34 4    1 A 4::a1Q3i0!AqrE(;CAa1fI KK$((9"55 IIy!AqrE(+ 3 DNN1% %r/) rSrTrUrVr"rv LayerNormr!rPrXrYs@r.rrs6K gg||'B r/rceZdZUejj eed< dfd ZdZ ejjdZ xZ S)XCA fused_attnct|||_td|_t j tj|dd|_ t j||dz||_ t j||_ t j|||_t j||_y)NT) experimentalrr:)r])r r!rrrr"rr=rr(LinearqkvDropoutrrlr)r,r*rrrrr-s r.r!z XCA.__init__s "(d;<< 9a(CD99S#'9I.IIc3' I.r/c|j\}}}|j|j||d|j||jzj ddddd}|j d\}}}|j rtjjj|d|jz}tjjj|d}tjjj|||d }ntjjj|d}tjjj|d}||jd dz|jz} | jd} |j| } | |z}|j ddddj|||}|j!|}|j#|}|S) Nr:rrr8rr5r9r)r')rqrrrrIunbindrr=r" functional normalizer(scaled_dot_product_attentionrrsoftmaxrrlr) r,rsr0rrrqkvrs r.rPz XCA.forwards''1ahhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1a ??##--aR-84;K;KKA##--aR-8A##@@AqPS@TA##--aR-8A##--aR-8A B++t/?/??D<>$'DqA IIaAq ! ) )!Q 2 IIaL NN1 r/cdhS)Nr(r,s r.no_weight_decayzXCA.no_weight_decays r/)rhFrr) rSrTrUr=jitFinalbool__annotations__r!rPignorerrXrYs@r.rrsC %% /0 YYr/rcheZdZdddddejej dffd ZdedefdZxZ S) XCABlockrFrrc pt || ||_t||||||_|dkDr t |nt j|_| ||_ t|||_ | ||_ t|t||z|||_t j | t#j$|z|_t j | t#j$|z|_t j | t#j$|z|_y)Nrr)rror)r r!rrrr r"rrnorm3rxlocal_mprrrWrrr=rrgamma3r) r,r*rrrrrrrorrr-s r.r!zXCABlock.__init__s _ y8ydmn 09B),BKKM_ yA _ 3C)O8LXahqrll3C#89 ll3C#89 ll3C#89 r/r1r2c ||j|j|j|j|zz}||j|j|j |j |||zz}||j|j|j|j|zz}|SN) rrrrrrrrrr)r,rsr1r2s r.rPzXCABlock.forwards t{{TYYtzz!}-EEF F t{{T]]4::a=!Q-OOP P t{{TXXdjjm-DDE Er/) rSrTrUr"rvrr!rWrPrXrYs@r.rrs= gg||:8CCr/rcfeZdZdZ dfd ZdZejjdZ ejjddZ ejjddZ ejjde jfdZdd ed eefd Z dd ej&d eeeeefdededededeeej&eej&eej&fffdZ dd eeeefdedefdZdZddefdZdZxZS) rz Based on timm and DeiT code bases https://github.com/rwightman/pytorch-image-models/tree/master/timm https://github.com/facebookresearch/deit/ ct||dvsJt|}|d|zdk(r |d|zdk(sJd|xsttj d}|xstj }||_|x|_x|_ |_ ||_ d|_ t||||||_|}t jt!j"dd||_|rt'| |_nd |_t j*| |_t j.t1|Dcgc]}t3||| | | ||||| c}|_t1|Dcgc]}t7||d |c}|_t j.t1|Dcgc]}t;||| | | ||||| c}|_|||_t j*| |_ |dkDr t jB|j|nt jD|_#tI|j$d|jK|jLy cc}wcc}wcc}w)a Args: img_size (int, tuple): input image size patch_size (int): patch size in_chans (int): number of input channels num_classes (int): number of classes for classification head embed_dim (int): embedding dimension depth (int): depth of transformer num_heads (int): number of attention heads mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP pos_drop_rate: position embedding dropout rate proj_drop_rate (float): projection dropout rate attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate (constant across all layers) norm_layer: (nn.Module): normalization layer cls_attn_layers: (int) Depth of Class attention layers use_pos_embed: (bool) whether to use positional encoding eta: (float) layerscale initialization value tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA Notes: - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch interaction (class LPI) and the patch embedding (class ConvPatchEmbed) avgtokenrz2`patch_size` should divide image dimensions evenlyr)r+F)rirjrmrnrorr9N)p) r*rrrrrrrorrzblocks.)num_chs reductionmodule) r*rrrrrrorrr{Gz?std)'r r!r rr"rrv num_classes num_featureshead_hidden_sizern global_poolgrad_checkpointingrd patch_embedrr=zerosrr pos_embedrpos_drop ModuleListrangerblocksdict feature_inforcls_attn_blocksnorm head_droprrheadr apply _init_weights)r,rirjrmrrrndepthrrr drop_rate pos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterorcls_attn_layers use_pos_embedrrr_ir-s r.r!z Xcit.__init__!sdd 2222X& j(A-HQK*4LPQ4Q A @ ARB72<!!!&&!,-?' $r/c ddhS)Nrrrrs r.rzXcit.no_weight_decays [))r/c"tddddgS)Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)i)stemrr)r)r,coarses r. group_matcherzXcit.group_matchers4$?AUV  r/c||_yr)r)r,enables r.set_grad_checkpointingzXcit.set_grad_checkpointings "(r/returnc|jSr)rrs r.get_classifierzXcit.get_classifiers yyr/rrc||_| |dvsJ||_|dkDr&tj|j||_ytj |_y)Nrr)rrr"rrrr)r,rrs r.reset_classifierzXcit.reset_classifiersU&  ""66 66*D ALqBIId//= VXVaVaVc r/rsindicesr stop_early output_fmtintermediates_onlyc |dvsJd|dk(}g}tt|j|\} } |j\} } } }|j |\}\}}|j G|j | ||j | d|jdjddd}||z}|j|}tjjs|s |j}n|jd| dz}t|D]q\}}|jr-tjjst||||}n ||||}|| vsN|j|r|j!|n|s|rA|Dcgc]6}|j | ||djdd ddj#8}}|r|Stj$|j&j)| dd|fd }|j*D]A}|jr+tjjs t||}:||}C|j!|}||fScc}w) a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to all intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWNLCz)Output format must be one of NCHW or NLC.r Nr5rrrr:r9)rlenrrqrrrrIrr=r is_scripting enumeraterrappendr contiguousrHrexpandr)r,rsrrr r r r intermediates take_indices max_indexr0rheightwidthrtru pos_encodingrrblkys r.forward_intermediateszXcit.forward_intermediatessA*_,Y.YY,& "6s4;;7G"Q i gg1fe&&q) 8B >> %>>!R4<> %>>!R4<* "81$1 r/c d|vr|d}t|dddu}|Dcgc]}|jds|}}|D],}|r%|j|||jdd<*||=.d|vrd|j vrt |j }t|D]}|jd|d}|jd d |jd }td D]\}} |||d|d | d <|jd|dd} | s| jd d } td D]\}} | ||d|d | d<|Scc}w)Nmodelrz pos_embeder.z pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightr:r5rz.attn.z.weightz.attn.qkv.biasz.bias) getattr startswithpopreplace state_dictrrrrrqr) r5r0rrpos_embed_keys num_ca_blocksr qkv_weightj subscriptrs r.checkpoint_filter_fnr;6s*( E;5TAM!+IAq||K/HaINI  BL..QRBSJqyy> ?1 +j8=^bgbrbrbt=tE112 }% [A#*:1#=M(NOJ#++Ar:3C3CB3GHJ )% 0 [ 9MWXY] -aSykIJ [!~~(8>&JDQH##++Ar2$-e$4[LAyOWXY{J!1!F9+UKL[ [ )Js E E c r|jdd}tt||ftt |dd|}|S)N out_indicesr:getter)r= feature_cls)pretrained_filter_fn feature_cfg)r3rrr;r)variant pretrained default_cfgkwargsr=r0s r. _create_xcitrFSsJ**]A.K   2[hG     E Lr/c 2|ddddddttddd |S) Nr))r:rererbicubicTzpatch_embed.proj.0.0r) urlr input_size pool_sizecrop_pct interpolationfixed_input_sizer&r first_conv classifierr)rIrEs r._cfgrQ`s2=t)%.B,F     r/zxcit_nano_12_p16_224.fb_in1kztimm/zhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthz#xcit_medium_24_p16_224.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthz#xcit_medium_24_p16_384.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthzxcit_large_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthz"xcit_large_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthz"xcit_large_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthzxcit_nano_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz xcit_nano_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz xcit_nano_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthzxcit_tiny_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz xcit_tiny_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz xcit_tiny_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthzxcit_tiny_24_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz xcit_tiny_24_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz xcit_tiny_24_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthzxcit_small_12_p8_224.fb_in1kzB!RVXJ  fZ f4PZKe^dKe fE Lr/c Vtdddddd}td d|it|fi|}|S) NrfrSr*rhrTr\rC)xcit_small_12_p16_384r^r_s r.rmrmrkr/c Vtdddddd}td d|it|fi|}|S) Nrfrdr8h㈵>Tr\rC)xcit_tiny_24_p16_224r^r_s r.rqrq =B!SWYJ  eJ e$zJd]cJd eE Lr/c Vtdddddd}td d|it|fi|}|S) Nrfrdror8rpTr\rC)xcit_tiny_24_p16_384r^r_s r.rtrt(rrr/c Vtdddddd}td d|it|fi|}|S) NrfrSrorhrpTr\rC)xcit_small_24_p16_224r^r_s r.rvrv0>B!SWYJ  fZ f4PZKe^dKe fE Lr/c Vtdddddd}td d|it|fi|}|S) NrfrSrorhrpTr\rC)xcit_small_24_p16_384r^r_s r.ryry8rwr/c Vtdddddd}td d|it|fi|}|S) NrfrorhrpTr\rC)xcit_medium_24_p16_224r^r_s r.r|r|@>B!SWYJ  gj gDQ[Lf_eLf gE Lr/c Vtdddddd}td d|it|fi|}|S) Nrfr{rorhrpTr\rC)xcit_medium_24_p16_384r^r_s r.rrHr}r/c Vtdddddd}tdd|it|fi|}|S) NrfrRrorpTr\rC)xcit_large_24_p16_224r^r_s r.rrP>B"$TXZJ  fZ f4PZKe^dKe fE Lr/c Vtdddddd}tdd|it|fi|}|S) NrfrRrorpTr\rC)xcit_large_24_p16_384r^r_s r.rrXrr/c Vtdddddd}td d|it|fi|}|S) Nrhr[r*r8rFr\rC)xcit_nano_12_p8_224r^r_s r.rra=2QVXJ  d: djIc\bIc dE Lr/c Vtdddddd}td d|it|fi|}|S) Nrhr[r*r8rFr\rC)xcit_nano_12_p8_384r^r_s r.rrirr/c Vtdddddd}td d|it|fi|}|S) Nrhrdr*r8rTr\rC)xcit_tiny_12_p8_224r^r_s r.rrq=2QUWJ  d: djIc\bIc dE Lr/c Vtdddddd}td d|it|fi|}|S) Nrhrdr*r8rTr\rC)xcit_tiny_12_p8_384r^r_s r.rryrr/c Vtdddddd}tdd|it|fi|}|S) NrhrSr*rTr\rC)xcit_small_12_p8_224r^r_s r.rr=2QUWJ  eJ e$zJd]cJd eE Lr/c Vtdddddd}tdd|it|fi|}|S) NrhrSr*rTr\rC)xcit_small_12_p8_384r^r_s r.rrrr/c Vtdddddd}td d|it|fi|}|S) Nrhrdror8rpTr\rC)xcit_tiny_24_p8_224r^r_s r.rr=2RVXJ  d: djIc\bIc dE Lr/c Vtdddddd}td d|it|fi|}|S) Nrhrdror8rpTr\rC)xcit_tiny_24_p8_384r^r_s r.rrrr/c Vtdddddd}tdd|it|fi|}|S) NrhrSrorpTr\rC)xcit_small_24_p8_224r^r_s r.rr=2RVXJ  eJ e$zJd]cJd eE Lr/c Vtdddddd}tdd|it|fi|}|S) NrhrSrorpTr\rC)xcit_small_24_p8_384r^r_s r.rrrr/c Vtdddddd}tdd|it|fi|}|S) Nrhr{rorpTr\rC)xcit_medium_24_p8_224r^r_s r.rr>2RVXJ  fZ f4PZKe^dKe fE Lr/c Vtdddddd}tdd|it|fi|}|S) Nrhr{rorpTr\rC)xcit_medium_24_p8_384r^r_s r.rrrr/c Vtdddddd}td d|it|fi|}|S) NrhrRrorfrpTr\rC)xcit_large_24_p8_224r^r_s r.rr=2SWYJ  eJ e$zJd]cJd eE Lr/c Vtdddddd}td d|it|fi|}|S) NrhrRrorfrpTr\rC)xcit_large_24_p8_384r^r_s r.rrrr/xcit_nano_12_p16_224_distxcit_nano_12_p16_384_distxcit_tiny_12_p16_224_distxcit_tiny_12_p16_384_distxcit_tiny_24_p16_224_distxcit_tiny_24_p16_384_distxcit_small_12_p16_224_distxcit_small_12_p16_384_distxcit_small_24_p16_224_distxcit_small_24_p16_384_distxcit_medium_24_p16_224_distxcit_medium_24_p16_384_distxcit_large_24_p16_224_distxcit_large_24_p16_384_distxcit_nano_12_p8_224_distxcit_nano_12_p8_384_distxcit_tiny_12_p8_224_distrTrUrVrWrXrY) xcit_tiny_12_p8_384_distxcit_tiny_24_p8_224_distxcit_tiny_24_p8_384_distxcit_small_12_p8_224_distxcit_small_12_p8_384_distxcit_small_24_p8_224_distxcit_small_24_p8_384_distxcit_medium_24_p8_224_distxcit_medium_24_p8_384_distxcit_large_24_p8_224_distxcit_large_24_p8_384_dist)r)FN)rr+)NrVr% functoolsrtypingrrrrr=torch.nnr" timm.datar r timm.layersr r r rr_builderr _featuresr _features_fxr _manipulater _registryrrrcaitr__all__r,rrbrdrxrrrrr;rFrQ default_cfgsr]rbrerhrjrmrqrtrvryr|rrrrrrrrrrrrrrrrrrSrr/r.rs5   // AOO*+1#YY (& &&D$RYY$N"))>1"))1h+"))+\#ryy#LY299Yx: %B&"D J%LB& ( O*Q B&( O\i*kB&#D J%LB&( O*QB&"( O\i*k#B&(#D J%L)B&.( O*Q/B&4( O\i*k5B&:$T K&M;B&@)$ P+RAB&F)$ P]j+lGB&L$T K&MMB&R)$ P+RSB&X)$ P]j+lYB&^%d L'N_B&d*4 Q,SeB&j*4 Q^k,mkB&p$T K&MqB&v)$ P+RwB&|)$ P]j+l}B&F"4 I$KGB&L' N)PMB&R' N[h)jSB&X"4 I$KYB&^' N)P_B&d' N[h)jeB&j"4 I$KkB&p' N)PqB&v' N[h)jwB&|#D J%L}B&B( O*QCB&H( O\i*kIB&N#D J%LOB&T*. O*Q*. O\i*k&* K&M+/ P+R+/ P]j+l%) J%L*. O*Q*. O\i*kB&B J$$ttttttH '!D ' !D ' !D '  !D '  !D ' !D '!"F '!"F '!"F '!"F '"#H '"#H '!"F '!"F '$ B% '& B' '( B) '*!C B B!D!D!D!D"F"F!D!D? ' r/