kh̿ndZddlZddlmZmZmZmZmZmZm Z m Z m Z ddl Z ddl mZddlmcmZddlmZmZddlmZmZmZmZmZmZmZmZmZmZddl m!Z!ddl"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+d gZ,e e-e e-e-ffZ.d e j^de e-e-fde j^fdZ0e%de j^de e-e-fde e-e-fde j^fdZ1GddejdZ3GddejdZ4GddejdZ5GddejdZ6Gdd ejdZ7dee8e j^fdejddee8e j^ffdZ9dNd e8d!e:de7fd"Z;dOd#Z<e)ee*dNd!e:de7fd<Z?e*dNd!e:de7fd=Z@e*dNd!e:de7fd>ZAe*dNd!e:de7fd?ZBe*dNd!e:de7fd@ZCe*dNd!e:de7fdAZDe*dNd!e:de7fdBZEe*dNd!e:de7fdCZFe*dNd!e:de7fdDZGe*dNd!e:de7fdEZHe*dNd!e:de7fdFZIe+eJdGdHdIdJdKdLdMy)PaK Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below Modifications and additions for timm hacked together by / Copyright 2022, Ross Wightman N) AnyCallableDictListOptionalSetTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPath to_2tuple trunc_normal_ClassifierHeadresample_patch_embedndgrid get_act_layer LayerType)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpoint)generate_default_cfgsregister_modelregister_model_deprecationsSwinTransformerV2x window_sizereturnc|j\}}}}|j|||dz|d||dz|d|}|jddddddjjd|d|d|}|S)zPartition into non-overlapping windows. Args: x: Input tensor of shape (B, H, W, C). window_size: Window size (height, width). Returns: Windows tensor of shape (num_windows*B, window_size[0], window_size[1], C). rrshapeviewpermute contiguous)r"r#BHWCwindowss [/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/swin_transformer_v2.pywindow_partitionr6$sJAq!Q q!{1~%{1~qKN7JKXYN\]^Aii1aAq)446;;B AP[\]P^`abG Nr4img_sizec|\}}|jd}|jd||dz||dz|d|d|}|jddddddjjd|||}|S)a1Merge windows back to feature map. Args: windows: Windows tensor of shape (num_windows * B, window_size[0], window_size[1], C). window_size: Window size (height, width). img_size: Image size (height, width). Returns: Feature map tensor of shape (B, H, W, C). r*rrr&r'r(r)r+)r4r#r8r1r2r3r"s r5window_reverser:4s DAq bA Rk!n,a;q>.A;q>S^_`SacdeA !Q1a#..055b!QBA Hr7ceZdZdZ ddedeeefdededededed eeefd d ffd Zdd Z deeefd d fdZ dde jde e jd e jfdZxZS)WindowAttentionzWindow based multi-head self attention (W-MSA) module with relative position bias. Supports both shifted and non-shifted window attention with continuous relative position bias and cosine attention. dimr# num_headsqkv_biasqkv_bias_separate attn_drop proj_droppretrained_window_sizer$Nc $t |||_||_t ||_||_||_tjtjdtj|ddfz|_ tjtjdddtj dtjd|d|_tj||d zd|_|rtjtj&||_|j+d tj&|d tjtj&||_nd |_d |_d |_tj0||_tj|||_tj0||_tj8d |_|j=y )a4Initialize window attention module. Args: dim: Number of input channels. window_size: The height and width of the window. num_heads: Number of attention heads. qkv_bias: If True, add a learnable bias to query, key, value. qkv_bias_separate: If True, use separate bias for q, k, v projections. attn_drop: Dropout ratio of attention weight. proj_drop: Dropout ratio of output. pretrained_window_size: The height and width of the window in pre-training. rr'iTbias)inplaceFr&k_bias persistentNr*r=)super__init__r=r#rrCr>r@nn Parametertorchlogones logit_scale SequentialLinearReLUcpb_mlpqkvzerosq_biasregister_bufferv_biasrIDropoutrAprojrBSoftmaxsoftmax"_make_pair_wise_relative_positions) selfr=r#r>r?r@rArBrC __class__s r5rNzWindowAttention.__init__Nsz. &&/0F&G#"!2<< "uzz9aQRBS7T2T(UV}} IIa4 ( GGD ! IIc95 1  99S#'6 ,,u{{3'78DK  5;;s+; N,,u{{3'78DKDKDKDKI.IIc3' I.zzb)  //1r7ctj|jddz |jdjtj}tj|jddz |jdjtj}tj t ||}|jdddjjd}|jddkDrO|dddddddfxx|jddz zcc<|dddddddfxx|jddz zcc<nN|dddddddfxx|jddz zcc<|dddddddfxx|jddz zcc<|dz}tj|tjtj|dzztjdz }|jd|d tj|jd}tj|jd}tj t ||}tj |d}|dddddf|dddddfz }|jdddj}|dddddfxx|jddz z cc<|dddddfxx|jddz z cc<|dddddfxxd|jdzdz zcc<|j#d } |jd | d y) z?Create pair-wise relative position index and coordinates table.rrr'N?relative_coords_tableFrJr*relative_position_index)rQaranger#tofloat32stackrr.r/ unsqueezerCsignlog2absmathr\flattensum) rcrelative_coords_hrelative_coords_wrhcoords_hcoords_wcoordscoords_flattenrelative_coordsris r5rbz2WindowAttention._make_pair_wise_relative_positionss"LL4+;+;A+>+B)CTEUEUVWEXY\\]b]j]jk!LL4+;+;A+>+B)CTEUEUVWEXY\\]b]j]jk % F3DFW,X Y 5 = =aA F Q Q S ] ]^_ `  & &q )A - !!Q1* -$2M2Ma2PST2T U - !!Q1* -$2M2Ma2PST2T U - !!Q1* -$2B2B12E2I J - !!Q1* -$2B2B12E2I J -" % +@ AEJJ II+ ,s 2E4!46:iil!C 46KX]^<< 0 0 34<< 0 0 34VHh78vq1(At4~aqj7QQ)11!Q:EEG1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? "1"5"5b"9 68O\abr7cht|}||jk7r||_|jyy)zUpdate window size and regenerate relative position tables. Args: window_size: New window size (height, width). N)rr#rb)rcr#s r5set_window_sizezWindowAttention.set_window_sizes5  , $** **D   3 3 5 +r7r"maskc|j\}}}|j|j|}ntj|j|j |j f}|jr|j|}||z }n,tj||jj|}|j||d|jdjddddd}|jd\}} } tj|dtj| dj!d dz} tj"|j$t'j(d  j+} | | z} |j-|j.j1d|j} | |j2j1dj1|j4d|j4dz|j4d|j4dzd}|jdddj7}d tj8|z}| |j;dz} ||jd}| j1d||j|||j;dj;dz} | j1d|j||} |j=| } n|j=| } |j?| } | | zj!ddj|||}|jA|}|jC|}|S) a#Forward pass of window attention. Args: x: Input features with shape of (num_windows*B, N, C). mask: Attention mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None. Returns: Output features with shape of (num_windows*B, N, C). )weightrGr&r*r'rrr(rLgY@)max)"r,r[rYrQcatrIr]r@Flinearrreshaper>r.unbind normalize transposeclamprTrrrRexprXrhr-rir#r/sigmoidrnrarAr_rB)rcr"r~B_Nr3rYr?qkvattnrTrelative_position_bias_tablerelative_position_biasnum_wins r5forwardzWindowAttention.forwards77Aq ;; ((1+Cyy$++t{{DKK!HIH%%hhqkxhhqxHkk"aDNNB7??1aAN**Q-1a A2&QB)?)I)I"b)QQkk$"2"28KLPPR k!'+||D4N4N'O'T'TUWY]YgYg'h$!=d>Z>Z>_>_`b>c!d!i!i   Q $"2"21"5 5t7G7G7JTM]M]^_M`7`bd"f!7!?!?1a!H!S!S!U!#emm4J&K!K,66q99  jjmG99R$..!Q?$..QRBSB]B]^_B``D99RA6D<<%D<<%D~~d# AX A & . .r1a 8 IIaL NN1 r7)TFr)rrr$NN)__name__ __module__ __qualname____doc__intr boolfloatrNrbr}rQTensorrr __classcell__rds@r5r<r<Gs"&+!!6<5252sCx52 52  52 $ 525252%*#s(O52 52nc> 65c? 6t 611Xell-C1u||1r7r<ceZdZdZddddddddddej df d ed ed ed ed ededede dede de de de de ejdeffd Z d$deej deej fdZ d$dedeedeeeefeeefffdZ d$d eeefd eeefdeeddfd!Zdej dej fd"Zdej dej fd#ZxZS)%SwinTransformerV2BlockzSwin Transformer V2 Block. A standard transformer block with window attention and shifted window attention for modeling long-range dependencies efficiently. rF@Trgelur=input_resolutionr>r# shift_sizealways_partition dynamic_mask mlp_ratior?rBrA drop_path act_layer norm_layerrCc "t|||_t||_||_t||_||_||_|j||\|_ |_ |jd|jdz|_ ||_ t| } t|t|j|| | | t||_|||_| dkDr t%| nt'j(|_t-|t/||z| | |_|||_| dkDr t%| nt'j(|_|j7d|jrdn|j9d y) a Args: dim: Number of input channels. input_resolution: Input resolution. num_heads: Number of attention heads. window_size: Window size. shift_size: Shift size for SW-MSA. always_partition: Always partition into full windows and shift mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Dropout rate. attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. pretrained_window_size: Window size in pretraining. rr)r#r>r?rArBrCr) in_featureshidden_featuresrdrop attn_maskNFrJ)rMrNr=rrr>target_shift_sizerr_calc_window_shiftr#r window_arearrr<rnorm1rrOIdentity drop_path1rrmlpnorm2 drop_path2r\ get_attn_mask)rcr=rr>r#rrrrr?rBrArrrrCrds r5rNzSwinTransformerV2Block.__init__skF  )*: ;"!*:!6 0(,0,C,CKQ[,\)$/++A.1A1A!1DD"!), # !$"2"23#,-C#D   _ 1:R(9-R[[]i0    _ 1:R(9-R[[]  %%D4+=+=+?  r7Nr"r$c t|jr|&tjdg|jd}nJtjd|j d|j ddf|j |j}d}d|jd f|jd |jd f|jd dffD]l}d|jd f|jd |jd f|jd dffD]$}||dd|d|d|d|dddf<|dz }&nt||j}|jd|j}|jd|jdz }|j|dk7tdj|dk(td}|Sd}|S) zGenerate attention mask for shifted window attention. Args: x: Input tensor for dynamic shape calculation. Returns: Attention mask or None if no shift. Nrr')dtypedevicerr*gYr)anyrrQrZrr,rrr#r6r-rrn masked_fillr)rcr"img_maskcnthw mask_windowsrs r5rz$SwinTransformerV2Block.get_attn_mask4s t y ;;'ED,A,A'E1'EF ;;1771:qwwqz1'EQWW]^]e]efC))!,,-&&q))DOOA,>+>?ooa(($/  T--a001**1--0B/BC//!,,d3A '>>$BRBRSTBUU!!!$q4+;+;A+>'>>$BRBRSTBUUHH''++I1a57QR   2r1%Y0@0@A NN2t'7'7;  4 /**95IIyyy; $((T-=-=a-@$BRBRSTBUWXY "<1A1AB8L a!RaRl+668   9T__6JAAr7c>|j\}}}}||j|j|j|z}|j |d|}||j |j |j|z}|j ||||}|S)Nr*)r,rrrrrrr)rcr"r0r1r2r3s r5rzSwinTransformerV2Block.forwardsWW 1a  4::a= 9: : IIaQ   488A; 78 8 IIaAq !r7r)rrrrrO LayerNormr_int_or_tuple_2_trrrr ModulerNrrQrrr rrrrrrs@r5rrs./,-%*!&!!!!!#)*,,,89!H H 0H  H + H * H #H H H H H H H !H RYYH %6!H T"x 5"%,,AW"N>B5 15 ((9:5 uS#Xc3h/ 0 5J04  S#X sCx 'tn    4,u||, ,\%,,r7rceZdZdZdej fdedeedeejffd Z de jde jfd Z xZS) PatchMergingzPatch Merging Layer. Merges 2x2 neighboring patches and projects to higher dimension, effectively downsampling the feature maps. Nr=out_dimrct|||_|xsd|z|_t j d|z|jd|_||j|_y)z Args: dim (int): Number of input channels. out_dim (int): Number of output channels (or 2 * dim if None) norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm r'r(FrFN)rMrNr=rrOrV reductionnorm)rcr=rrrds r5rNzPatchMerging.__init__sS )!c' 1s7DLLuEt||, r7r"r$ch|j\}}}}ddd|dzd|dzf}tjj||}|j\}}}}|j ||dzd|dzd|j ddddddj d}|j|}|j|}|S)Nrr'rr&r(r)) r,rOrrrr.rsrr)rcr"r0r1r2r3 pad_valuesrs r5rzPatchMerging.forwardsWW 1aAq1uaQ/ MM  a ,WW 1a IIaaAFAq 1 9 9!Q1a K S STU V NN1  IIaLr7)rrrrrOrrrr rrNrQrrrrs@r5rrs\&**,,, --c]-RYY -$  %,, r7rc'.eZdZdZdddddddddej ddf ded ed ed ed ed edededede dede de de de e e fde ejdededdf&fd Z d"deeefd edeeddfdZdej(dej(fd Zd#d!ZxZS)$SwinTransformerV2StagezA Swin Transformer V2 Stage. A single stage consisting of multiple Swin Transformer blocks with optional downsampling at the beginning. FrTrrrr=rrdepthr>r#rr downsamplerr?rBrArrrrC output_nchwr$NcPt|||_||_| rt d|Dn||_||_||_d|_t|}t |Dcgc]}|dz c}}| rt||||_ n ||k(sJtj|_ tjt|Dcgc]E}t!||j |||dzdk(rdn|||| | | | t#|t$r||n||||Gc}|_ycc}wcc}w)a  Args: dim: Number of input channels. out_dim: Number of output channels. input_resolution: Input resolution. depth: Number of blocks. num_heads: Number of attention heads. window_size: Local window size. always_partition: Always partition into full windows and shift dynamic_mask: Create attention mask in forward based on current input size downsample: Use downsample layer at start of the block. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Projection dropout rate attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer type. norm_layer: Normalization layer. pretrained_window_size: Local window size in pretraining. output_nchw: Output tensors on NCHW format instead of NHWC. c3&K|] }|dz ywr'N.0is r5 z2SwinTransformerV2Stage.__init__..$s&H!qAv&HFr')r=rrr)r=rr>r#rrrrr?rBrArrrrCN)rMrNr=rroutput_resolutionrrgrad_checkpointingrrrrOr ModuleListranger isinstancelistblocks)rcr=rrrr>r#rrrrr?rBrArrrrCrrrrrds r5rNzSwinTransformerV2Stage.__init__s/T  0LV&H7G&H!H\l &"' , K8qAF89  *sGPZ[DO'> !> kkmDOmm$5\%%#$# #!%!7!7#'!"Q!1*!1)#!##*4Y*E)A,9#%'= %#$ 9%#s " DA D#rc.||_t|jtjr||_n3t|jt sJtd|D|_|jD] }|j|j ||"y)zUpdate resolution, window size and relative positions. Args: feat_size: New input (feature) resolution. window_size: New window size. always_partition: Always partition / shift the window. c3&K|] }|dz ywrrrs r5rz8SwinTransformerV2Stage.set_input_size..Ys*Ea16*Errr#rN) rrrrOrrrrrr)rcrr#rblocks r5rz%SwinTransformerV2Stage.set_input_sizeGs!* door{{ 3%.D "doo|< <<%**E9*E%ED "[[ E  00'!1 !  r7r"c|j|}|jD]A}|jr+tjj s t ||}:||}C|S)zForward pass through the stage. Args: x: Input tensor of shape (B, H, W, C). Returns: Output tensor of shape (B, H', W', C'). )rrrrQjit is_scriptingr)rcr"blks r5rzSwinTransformerV2Stage.forwardasY OOA ;; C&&uyy/E/E/GsA&F   r7c|jD]}tjj|jj dtjj|jj dtjj|jj dtjj|jj dy)z/Initialize residual post-normalization weights.rN)rrOinit constant_rrGrr)rcr s r5_init_respostnormz(SwinTransformerV2Stage._init_respostnormss;; 3C GG  ciinna 0 GG  cii.. 2 GG  ciinna 0 GG  cii.. 2  3r7rr)rrrrrOrrrrrr strrr rrNr rrrQrrrrrs@r5rrsu&+!&$!!!!!.4*,,,89 %'N$N$N$0 N$  N$  N$+N$#N$N$N$N$N$N$N$N$ S(]+!N$"RYY#N$$%6%N$&'N$( )N$h04 S#X'tn   4%,,$3r7rc+6eZdZdZdddddddd d d d d d dddddej dfdedededededede edfde edfdede de de de d e d!e d"e d#e d$e ee fd%e d&e edff(fd' Zd(ejd)d*fd+Z dCdee eefdee eefdee eefd,eedee f d-Zej(j*d)eefd.Zej(j*dDd/e d)eeeffd0Zej(j*dEd1e d)d*fd2Zej(j*d)ejfd3ZdFdedeed)d*fd4Z dGd5ej<d6ee eeefd7e d8e d9ed:e d)e eej<e ej<eej<fffd;Z dHd6e eeefdZ!d5ej<d)ej<fd?Z"dDd5ej<d@e d)ej<fdAZ#d5ej<d)ej<fdBZ$xZ%S)Ir!a Swin Transformer V2. A hierarchical vision transformer using shifted windows for efficient self-attention computation with continuous position bias. A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 r(r&avg`r'r'r'r&r rFTrrg?r)rrrrr8 patch_sizein_chans num_classes global_pool embed_dimdepths.r>r#rstrict_img_sizerr? drop_rateproj_drop_rateattn_drop_ratedrop_path_raterrpretrained_window_sizesc t|||_|dvsJ||_d|_t ||_||_t|d|j dz zzx|_ |_ g|_ t|ttfs1t|j Dcgc]}t|d|zz}}t!||||d|| d|_|j"j$}t'j(d|t+|j-|Dcgc]}|j/}}g}|d}d}t|j D]}||}|t1did|d|d |d|z|d|zfd ||d |dkDd ||d | d| d| d| d| d|d|d||d|d|d||gz }|}|dkDr|dz}|xjt3|d|zd|gz c_ t5j6||_||j|_t=|j||||j|_|jA|jB|j8D]}|jEycc}wcc}w)a] Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of input image channels. num_classes: Number of classes for classification head. embed_dim: Patch embedding dimension. depths: Depth of each Swin Transformer stage (layer). num_heads: Number of attention heads in different layers. window_size: Window size. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. drop_rate: Head dropout rate. proj_drop_rate: Projection dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. norm_layer: Normalization layer. act_layer: Activation layer type. patch_norm: If True, add normalization after patch embedding. pretrained_window_sizes: Pretrained window sizes of each layer. output_fmt: Output tensor format if not None, otherwise output 'NHWC' by default. )rNHWCr'rr)r8rrr!rr# output_fmtr=rrrrr>r#rrrr?rBrArrrrCr(layers.)num_chsrmodule) pool_typer$ input_fmtNr)#rMrNrr r,len num_layersr!r num_featureshead_hidden_size feature_inforrrrr patch_embed grid_sizerQlinspacertsplittolistrdictrOrUlayersrrheadapply _init_weightsr) rcr8rrrr r!r"r>r#rr#rr?r$r%r&r'rrr(kwargsrr8r"dprr=in_dimscalerblyrds r5rNzSwinTransformerV2.__init__s\ &k)))& f+"47 A$//\]J]D^8^4__D1)eT]3:?:PQQYa/0QIQ&!l!+ $$.. #(>>!^S[#Q#W#WX^#_`aqxxz``1t' dAlG -#,A,%"719N!OQi  q5  $A, ("2"10$"))a&$ &!"(?q'A# F&F1u    $w!e)V]^_]`Ta"b!c c 1 d4mmV, t001 "    !oo    4%%&;; $C  ! ! # $oRas JJmr$Nc t|tjrjt|jdt|tjr8|j +tj j|j dyyyy)zVInitialize model weights. Args: m: Module to initialize. g{Gz?)stdNr)rrOrVrrrGrr)rcrFs r5r@zSwinTransformerV2._init_weightss\ a # !(( ,!RYY'AFF,>!!!&&!,-?' $r7 window_ratioc^||3|jj|||jj}||tDcgc]}||z c}}t |j D]9\}} dt |dz dz} | jd| z|d| zf||;ycc}w)aTUpdates the image resolution, window size, and so the pair-wise relative positions. Args: img_size (Optional[Tuple[int, int]]): New input resolution, if None current resolution is used patch_size (Optional[Tuple[int, int]): New patch size, if None use current patch size window_size (Optional[int]): New window size, if None based on new_img_size // window_div window_ratio (int): divisor for calculating window size from patch grid size always_partition: always partition / shift windows even if feat size is < window N)r8rr'rrr)r7rr8r enumerater=r) rcr8rr#rIrr8rindexstage stage_scales r5rz SwinTransformerV2.set_input_sizes"  :#9    + +X* + U((22I  <#;I Fql!2 FGK%dkk2 LE5s519a00K  $Q<;6 ! 8ST'!1 !  !Gs B*ct}|jD]2\}}tdDcgc]}||vc}s"|j|4|Scc}w)zGet parameter names that should not use weight decay. Returns: Set of parameter names to exclude from weight decay. )rXrT)set named_modulesradd)rcnodnrFkws r5no_weight_decayz!SwinTransformerV2.no_weight_decay%sWe&&( DAq&@AB!GAB   Bs A coarsec2td|rdSgdS)zCreate parameter group matcher for optimizer parameter groups. Args: coarse: If True, use coarse grouping. Returns: Dictionary mapping group names to regex patterns. z^absolute_pos_embed|patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsample)r)z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)i)stemr)r<)rcrWs r5 group_matcherzSwinTransformerV2.group_matcher2s)3(.$  5  r7enablec4|jD] }||_ y)z}Enable or disable gradient checkpointing. Args: enable: If True, enable gradient checkpointing. N)r=r)rcr[ls r5set_grad_checkpointingz(SwinTransformerV2.set_grad_checkpointingEs *A#)A  *r7c.|jjS)z_Get the classifier head. Returns: The classification head module. )r>fc)rcs r5get_classifierz SwinTransformerV2.get_classifierOsyy||r7cJ||_|jj||y)zReset the classification head. Args: num_classes: Number of classes for new head. global_pool: Global pooling type. N)rr>reset)rcrr s r5reset_classifierz"SwinTransformerV2.reset_classifierXs'  [1r7r"indicesr stop_earlyr,intermediates_onlyc>|dvsJdg}tt|j|\}} |j|}t|j} tj j s|s |j} n|jd| dz} t| D]c\} } | |}| |vs|r| | dz k(r|j|}n|}|jddddj}|j|e|r|S|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.Nrrr&r') rr2r=r7rQr r rKrr.r/append)rcr"rerrfr,rg intermediates take_indices max_index num_stagesstagesrrMx_inters r5forward_intermediatesz'SwinTransformerV2.forward_intermediatesbs*Y&D(DD& "6s4;;7G"Q i   Q % 99 ! ! #:[[F[[)a-0F!&) .HAuaAL Aa/"iilGG!//!Q15@@B$$W- .  IIaL-r7 prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr*)rr2r=rOrrrd)rcrerrrsrlrms r5prune_intermediate_layersz+SwinTransformerV2.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r7cl|j|}|j|}|j|}|S)zForward pass through feature extraction layers. Args: x: Input tensor of shape (B, C, H, W). Returns: Feature tensor of shape (B, H', W', C). )r7r=rrcr"s r5forward_featuresz"SwinTransformerV2.forward_featuress3   Q  KKN IIaLr7 pre_logitscN|r|j|dS|j|S)aForward pass through classification head. Args: x: Feature tensor of shape (B, H, W, C). pre_logits: If True, return features before final linear layer. Returns: Logits tensor of shape (B, num_classes) or pre-logits. T)ry)r>)rcr"rys r5 forward_headzSwinTransformerV2.forward_heads&1;tyyty,L ! Lr7cJ|j|}|j|}|S)zForward pass through the model. Args: x: Input tensor of shape (B, C, H, W). Returns: Logits tensor of shape (B, num_classes). )rxr{rws r5rzSwinTransformerV2.forwards)  ! !! $   a r7)NNNrfNF)Tr)NFFriF)rFT)&rrrrrOrrrrr rrr rrNrr@rrrQr ignorerrVrrrZr^rardrrrqrurxr{rrrs@r5r!r!|s+.#$&2)7-.%*$(!!!$&$&$'.4#%<<7C+r$'r$r$ r$  r$  r$r$#s(Or$S#Xr$+r$#r$"r$r$r$r$ "!r$""#r$$"%r$&S(]+'r$(!)r$*&+38_+r$h -ryy -T -374859*+/3 uS#X/!sCx1"%S/2  #3-  'tn @ YY S   YY D T#s(^  $ YY*T*T** YY 2C2hsm2W[28<$$',0 ||0 eCcN340  0  0  0 !%0  tELL!5tELL7I)I#JJ K0 h./$# 3S >*  %,, 5<<  Mell M M M  %,, r7 state_dictmodelc|jd|}|jd|}d|v}i}ddl}|jD]\}}tdDcgc]}||vc}r"d|vre|jj j j\}}} } |jd| k7s|jd | k7rt|| | fd d d }|s&|jd d|}|jdd}|||<|Scc}w)aMFilter and process checkpoint state dict for loading. Handles resizing of patch embeddings and relative position tables when model size differs from checkpoint. Args: state_dict: Checkpoint state dictionary. model: Target model to load weights into. Returns: Filtered state dictionary. rrzhead.fc.weightrN)rirhrzpatch_embed.proj.weightrr*bicubicT) interpolation antialiasverbosezlayers.(\d+).downsamplecDdt|jddzdS)Nr-rz .downsample)rgroup)r"s r5z&checkpoint_filter_fn..s$ws177ST:YZGZF[[f=gr7zhead.zhead.fc.) getreitemsrr7r_rr,rsubreplace) rrnative_checkpointout_dictrrrrTrr1r2s r5checkpoint_filter_fnrs4J j9J(J6H   "1  ab1Qb c  $ )**//66<.s\da\rr")rrrr out_indicesT)flatten_sequentialr)pretrained_filter_fn feature_cfg)rrKrpoprr!rr<)rrrAdefault_out_indicesrrs r5_create_swin_transformer_v2rsj \i 8\8Z.[\\**],?@K 7J 1DkJ    E Lr7c 4|ddddddttddd d |S) Nr)r&r)rfrfg?rTzpatch_embed.projzhead.fcmit) urlr input_size pool_sizecrop_pctrfixed_input_sizemeanrH first_conv classifierlicenser )rrAs r5_cfgr s5=v%.B(    #  r7ztimm/z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to16_192to256_22kto1k_ft.pth) hf_hub_idrz{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to24_192to384_22kto1k_ft.pth)r&r)rrrg)rrrrrz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to16_192to256_22kto1k_ft.pthz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to24_192to384_22kto1k_ft.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window16_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window8_256.pthzhhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window16_256.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window16_256.pthzkhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12_192_22k.pthiQU)r&r)rr)rrrrrzlhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12_192_22k.pth) 2swinv2_base_window12to16_192to256.ms_in22k_ft_in1k2swinv2_base_window12to24_192to384.ms_in22k_ft_in1k3swinv2_large_window12to16_192to256.ms_in22k_ft_in1k3swinv2_large_window12to24_192to384.ms_in22k_ft_in1kzswinv2_tiny_window8_256.ms_in1kz swinv2_tiny_window16_256.ms_in1kz swinv2_small_window8_256.ms_in1kz!swinv2_small_window16_256.ms_in1kzswinv2_base_window8_256.ms_in1kz swinv2_base_window16_256.ms_in1k!swinv2_base_window12_192.ms_in22k"swinv2_large_window12_192.ms_in22kc Ptdddd}t dd|it|fi|S)z"Swin-T V2 @ 256x256, window 16x16.rrrrr#r!r"r>r)swinv2_tiny_window16_256r<rrrA model_argss r5rrQsD"B:>XQW>X ZZr7c Ptdddd}t dd|it|fi|S)z Swin-S V2 @ 256x256, window 8x8.rfrrrrr)swinv2_small_window8_256rrs r5rrisD!r-SabJ &" Y/9 Y=A*=WPV=W YYr7c Ptdddd}t dd|it|fi|S)z"Swin-B V2 @ 256x256, window 16x16.rrr(rfr rr)swinv2_base_window16_256rrs r5rrqD"MUcdJ &" Y/9 Y=A*=WPV=W YYr7c Ptdddd}t dd|it|fi|S)z Swin-B V2 @ 256x256, window 8x8.rfrrrrr)swinv2_base_window8_256rrs r5rrysC!s=TbcJ &! X.8 X<@r(r)!swinv2_base_window12to16_192to256rrs r5rrK#m~ /1J '+ b8B bFJ:F`Y_F` bbr7c Rtddddd}t dd|it|fi|S) zQSwin-B V2 @ 192x192, trained at window 12x12, fine-tuned to 384x384 window 24x24.rrrrrrr)!swinv2_base_window12to24_192to384rrs r5rrrr7c Ptdddd}t dd|it|fi|S)z"Swin-L V2 @ 192x192, window 12x12.rrrrrr0rr)swinv2_large_window12_192rrs r5rrsD"MUdeJ &# Z0: Z>B:>XQW>X ZZr7c Rtddddd}t dd|it|fi|S) zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 256x256 window 16x16.rrrrrrr)"swinv2_large_window12to16_192to256rrs r5rrK#m /1J ', c9C cGKJGaZ`Ga ccr7c Rtddddd}t dd|it|fi|S) zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 384x384 window 24x24.rrrrrrr)"swinv2_large_window12to24_192to384rrs r5rrrr7rrrrrr)swinv2_base_window12_192_22k)swinv2_base_window12to16_192to256_22kft1k)swinv2_base_window12to24_192to384_22kft1kswinv2_large_window12_192_22k*swinv2_large_window12to16_192to256_22kft1k*swinv2_large_window12to24_192to384_22kft1kr})r*)Krrrtypingrrrrrrr r r rQtorch.nnrOtorch.nn.functionalrr timm.datar r timm.layersrrrrrrrrrr_builderr _featuresr _features_fxr _manipulater _registryrrr __all__rrrr6r:rr<rrrr!rrrrr default_cfgsrrrrrrrrrrrrrrr7r5rs OOO A;;;*+3#YY  #uS#X./  5c? u||   ELL uS#X RWX[]`X`Ra fkfrfr  $YbiiYxdRYYdN#299#LI3RYYI3XM M` 'T#u||*;%<'RYY'SWX[]b]i]iXiSj'T$Uf,%:> J;;? J Hs; <@ K<<@ K Hs< (, t()- u))- u)*. v*(, t()- u) *. ymv* +/ zmv+e7&7 tYYDUYYXXCTXXZ$ZEVZZYYDUYYYYDUYYXXCTXXYYDUYYb$bM^bbb$bM^bbZ$ZEVZZc4cN_ccc4cN_ccH$G1e1e%I2g2g 'r7