
    kh                     	   d Z ddlmZ ddlmZ ddlmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddl
mZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZmZ d
dlm Z  d
dl!m"Z" d
dl#m$Z$m%Z% d
dl&m'Z'm(Z( dgZ) G d dejT                        Z+ G d dejT                        Z, G d dejT                        Z- G d dejT                        Z. G d dejT                        Z/ G d dejT                        Z0 G d de      Z1 G d de      Z2 G d  d!ej6                        Z3 G d" d#ejT                        Z4 G d$ d%ejT                        Z5 G d& d'ejT                        Z6 G d( d)ejT                        Z7 G d* d+ejT                        Z8 G d, dejT                        Z9d- Z:dd.Z;dd/Z< e'i d0 e<d1d23      d4 e<d1d23      d5 e<d1d23      d6 e<d1d73      d8 e<d1d73      d9 e<d1:      d; e<d1:      d< e<d1:      d= e<d1:      d> e<d1:      d? e<d1d@A      dB e<d1d@dCdDE      dF e<d1d@A      dG e<d1d@dCdDE      dH e<d1d@dIJ      dK e<d1d@A      dL e<d1d@dCdDE      i dM e<d1d@A      dN e<d1d@dCdDE      dO e<d1d@dIJ      dP e<d1d@A      dQ e<d1d@dCdDE      dR e<d1d@A      dS e<d1d@dCdDE      dT e<d1d@dIJ      dU e<d1d@A      dV e<d1d@dCdDE      dW e<d1d@A      dX e<d1d@dCdDE      dY e<d1d@dIJ      dZ e<d1d@A      d[ e<d1d@dCdDE      d\ e<d1d@A      d] e<d1d@dCdDE      i d^ e<d1d@dIJ      d_ e<d1d@A      d` e<d1d@dCdDE      da e<d1d@A      db e<d1d@dCdDE      dc e<d1d@dIJ      dd e<d1d@A      de e<d1d@dCdDE      df e<d1d@A      dg e<d1d@dCdDE      dh e<d1d@dIJ      di e<d1d@A      dj e<d1d@dCdDE      dk e<d1d@A      dl e<d1d@dCdDE      dm e<d1d@dIJ            Z=e(ddne9fdo       Z>e(ddne9fdp       Z?e(ddne9fdq       Z@e(ddne9fdr       ZAe(ddne9fds       ZBe(ddne9fdt       ZCe(ddne9fdu       ZDe(ddne9fdv       ZEe(ddne9fdw       ZFe(ddne9fdx       ZGe(ddne9fdy       ZHe(ddne9fdz       ZIe(ddne9fd{       ZJe(ddne9fd|       ZKe(ddne9fd}       ZLe(ddne9fd~       ZMe(ddne9fd       ZNe(ddne9fd       ZOy)a  
Poolformer from MetaFormer is Actually What You Need for Vision https://arxiv.org/abs/2111.11418

IdentityFormer, RandFormer, PoolFormerV2, ConvFormer, and CAFormer
from MetaFormer Baselines for Vision https://arxiv.org/abs/2210.13452

All implemented models support feature extraction and variable input resolution.

Original implementation by Weihao Yu et al.,
adapted for timm by Fredo Guan and Ross Wightman.

Adapted from https://github.com/sail-sg/metaformer, original copyright below
    )OrderedDict)partial)ListOptionalTupleUnionN)Tensor)FinalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_DropPathSelectAdaptivePool2d
GroupNorm1	LayerNormLayerNorm2dMlpuse_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_model
MetaFormerc                   ,     e Zd ZdZ	 d fd	Zd Z xZS )Stemzc
    Stem implemented by a layer of convolution.
    Conv2d params constant across all models.
    c                     t         |           t        j                  ||ddd      | _        |r ||      | _        y t        j
                         | _        y )N         kernel_sizestridepadding)super__init__nnConv2dconvIdentitynorm)selfin_channelsout_channels
norm_layer	__class__s       R/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/metaformer.pyr)   zStem.__init__8   sK     	II
	 1;J|,		    c                 J    | j                  |      }| j                  |      }|S N)r,   r.   r/   xs     r4   forwardzStem.forwardH   !    IIaLIIaLr5   r7   __name__
__module____qualname____doc__r)   r:   __classcell__r3   s   @r4   r   r   2   s     	N r5   r   c                   0     e Zd ZdZ	 	 	 d fd	Zd Z xZS )Downsamplingz=
    Downsampling implemented by a layer of convolution.
    c                     t         |           |r ||      nt        j                         | _        t        j
                  |||||      | _        y )Nr$   )r(   r)   r*   r-   r.   r+   r,   )r/   r0   r1   r%   r&   r'   r2   r3   s          r4   r)   zDownsampling.__init__S   sE     	/9J{+r{{}	II#
	r5   c                 J    | j                  |      }| j                  |      }|S r7   )r.   r,   r8   s     r4   r:   zDownsampling.forwardf   r;   r5   )r   r   Nr<   rB   s   @r4   rD   rD   N   s     
&r5   rD   c                   *     e Zd ZdZd fd	Zd Z xZS )Scalez2
    Scale vector by element multiplications.
    c                     t         |           |r|ddfn|f| _        t        j                  |t        j                  |      z  |      | _        y )Nr   requires_grad)r(   r)   shaper*   	Parametertorchonesscale)r/   dim
init_value	trainableuse_nchwr3   s        r4   r)   zScale.__init__q   s@    $,c1a[3&
\\*uzz#">iX
r5   c                 R    || j                   j                  | j                        z  S r7   )rP   viewrL   r8   s     r4   r:   zScale.forwardv   s    4::??4::...r5   )      ?TTr<   rB   s   @r4   rH   rH   l   s    Y
/r5   rH   c                   *     e Zd ZdZd fd	Zd Z xZS )SquaredReLUz<
        Squared ReLU: https://arxiv.org/abs/2109.08668
    c                 X    t         |           t        j                  |      | _        y )Ninplace)r(   r)   r*   ReLUrelu)r/   r\   r3   s     r4   r)   zSquaredReLU.__init__   s    GGG,	r5   c                 J    t        j                  | j                  |            S r7   )rN   squarer^   r8   s     r4   r:   zSquaredReLU.forward   s    ||DIIaL))r5   Fr<   rB   s   @r4   rY   rY   z   s    -*r5   rY   c                   6     e Zd ZdZ	 	 	 	 	 	 d fd	Zd Z xZS )StarReLUz(
    StarReLU: s * relu(x) ** 2 + b
    c                 .   t         |           || _        t        j                  |      | _        t        j                  |t        j                  d      z  |      | _	        t        j                  |t        j                  d      z  |      | _
        y )Nr[   r   rJ   )r(   r)   r\   r*   r]   r^   rM   rN   rO   rP   bias)r/   scale_value
bias_valuescale_learnablebias_learnablemoder\   r3   s          r4   r)   zStarReLU.__init__   se     	GGG,	\\+

1"=_]
LLejjm!;>Z	r5   c                 ^    | j                   | j                  |      dz  z  | j                  z   S )Nr#   )rP   r^   re   r8   s     r4   r:   zStarReLU.forward   s'    zzDIIaLA--		99r5   )rW           TTNFr<   rB   s   @r4   rc   rc      s'      [:r5   rc   c                   H     e Zd ZU dZee   ed<   	 	 	 	 	 	 d fd	Zd Z xZ	S )	Attentionzl
    Vanilla self-attention from Transformer: https://arxiv.org/abs/1706.03762.
    Modified from timm.
    
fused_attnc                    t         	|           || _        |dz  | _        t	               | _        |r|n||z  | _        | j                  dk(  rd| _        | j                  | j                  z  | _        t        j                  || j                  dz  |      | _
        t        j                  |      | _        t        j                  | j                  ||      | _        t        j                  |      | _        y )Ng      r   r      re   )r(   r)   head_dimrP   r   ro   	num_headsattention_dimr*   LinearqkvDropout	attn_dropproj	proj_drop)
r/   rQ   rs   rt   qkv_biasry   r{   	proj_biaskwargsr3   s
            r4   r)   zAttention.__init__   s     	 %
(*&/SH_>>QDN!^^dmm;99S$"4"4q"8xHI.IId00#IF	I.r5   c                    |j                   \  }}}| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }}}| j                  r<t        j                  |||| j                  r| j                  j                  nd      }nJ||j                  dd	      z  | j                  z  }	|	j                  d	
      }	| j                  |	      }	|	|z  }|j                  dd      j                  |||      }| j!                  |      }| j#                  |      }|S )Nrq   r#   r   r   r"   rl   )	dropout_p)rQ   )rL   rw   reshapert   rs   permuteunbindro   Fscaled_dot_product_attentiontrainingry   p	transposerP   softmaxrz   r{   )
r/   r9   BNCrw   qkvattns
             r4   r:   zAttention.forward   s   ''1ahhqk!!!Q4>>4==IQQRSUVXY[\^_`**Q-1a??..1a.2mm$..**A
 B++tzz9D<<B<'D>>$'DqAKK1%%aA.IIaLNN1r5   )    NFrl   rl   F)
r=   r>   r?   r@   r
   bool__annotations__r)   r:   rA   rB   s   @r4   rn   rn      s2     d
 /8r5   rn   c                        e Zd Z fdZ xZS )GroupNorm1NoBiasc                 b    t        |   |fi | |j                  dd      | _        d | _        y Nepsư>r(   r)   getr   re   r/   num_channelsr~   r3   s      r4   r)   zGroupNorm1NoBias.__init__   .    00::eT*	r5   r=   r>   r?   r)   rA   rB   s   @r4   r   r           r5   r   c                        e Zd Z fdZ xZS )LayerNorm2dNoBiasc                 b    t        |   |fi | |j                  dd      | _        d | _        y r   r   r   s      r4   r)   zLayerNorm2dNoBias.__init__   r   r5   r   rB   s   @r4   r   r      r   r5   r   c                        e Zd Z fdZ xZS )LayerNormNoBiasc                 b    t        |   |fi | |j                  dd      | _        d | _        y r   r   r   s      r4   r)   zLayerNormNoBias.__init__   r   r5   r   rB   s   @r4   r   r      r   r5   r   c                   J     e Zd ZdZdeej                  dddf fd	Zd Z xZ	S )SepConvz\
    Inverted separable convolution from MobileNetV2: https://arxiv.org/abs/1801.04381.
    r#   Fr!   rq   c                 &   t         
|           t        ||z        }	t        j                  ||	d|      | _         |       | _        t        j                  |	|	|||	|      | _         |       | _        t        j                  |	|d|      | _	        y )Nr   )r%   re   )r%   r'   groupsre   )
r(   r)   intr*   r+   pwconv1act1dwconvact2pwconv2)r/   rQ   expansion_ratio
act1_layer
act2_layerre   r%   r'   r~   mid_channelsr3   s             r4   r)   zSepConv.__init__   s}     	?S01yylML	ii,KLt= L	yysMr5   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r7   )r   r   r   r   r   r8   s     r4   r:   zSepConv.forward
  sH    LLOIIaLKKNIIaLLLOr5   )
r=   r>   r?   r@   rc   r*   r-   r)   r:   rA   rB   s   @r4   r   r      s+     {{N*r5   r   c                   *     e Zd ZdZd fd	Zd Z xZS )PoolingzT
    Implementation of pooling for PoolFormer: https://arxiv.org/abs/2111.11418
    c                 d    t         |           t        j                  |d|dz  d      | _        y )Nr   r#   F)r&   r'   count_include_pad)r(   r)   r*   	AvgPool2dpool)r/   	pool_sizer~   r3   s      r4   r)   zPooling.__init__  s,    LLaa5R	r5   c                 .    | j                  |      }||z
  S r7   )r   )r/   r9   ys      r4   r:   zPooling.forward  s    IIaL1ur5   )rq   r<   rB   s   @r4   r   r     s    R
r5   r   c                   6     e Zd ZdZddeeddf fd	Zd Z xZS )MlpHeadz MLP classification head
      r"   rl   Tc                    t         	|           t        ||z        }t        j                  |||      | _         |       | _         ||      | _        t        j                  |||      | _        t        j                  |      | _
        y )Nrr   )r(   r)   r   r*   rv   fc1actr.   fc2rx   	head_drop)
r/   rQ   num_classes	mlp_ratio	act_layerr2   	drop_ratere   hidden_featuresr3   s
            r4   r)   zMlpHead.__init__&  sk     	i#o.99S/=;/	99_kEI.r5   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r7   )r   r   r.   r   r   r8   s     r4   r:   zMlpHead.forward8  sI    HHQKHHQKIIaLNN1HHQKr5   )	r=   r>   r?   r@   rY   r   r)   r:   rA   rB   s   @r4   r   r   "  s&     ! /$r5   r   c            	       <     e Zd ZdZeededddddf	 fd	Zd Z xZ	S )MetaFormerBlockz1
    Implementation of one MetaFormer block.
    Frl   TNc                    t         |           t        t        ||	|      }t        t        ||
|      } ||      | _         |d||d|| _        |dkD  rt        |      nt        j                         | _	        |	 |       nt        j                         | _
        |
 |       nt        j                         | _         ||      | _        t        |t        d|z        ||||      | _        |dkD  rt        |      nt        j                         | _        |	 |       nt        j                         | _        |
 |       | _        y t        j                         | _        y )N)rQ   rR   rT   )rQ   r{   rl   r"   )r   re   dropuse_conv )r(   r)   r   rH   norm1token_mixerr   r*   r-   
drop_path1layer_scale1
res_scale1norm2r   r   mlp
drop_path2layer_scale2
res_scale2)r/   rQ   r   mlp_actmlp_biasr2   r{   	drop_pathrT   layer_scale_init_valueres_scale_init_valuer~   ls_layerrs_layerr3   s                 r4   r)   zMetaFormerBlock.__init__F  s#    	5c6LW_`5c6JU]^_
&N3)NvN1:R(9-R[[]*@*LHJRTR]R]R_(<(H(*bkkm_
CL
 2;R(9-R[[]*@*LHJRTR]R]R_(<(H(*bkkmr5   c           
      F   | j                  |      | j                  | j                  | j                  | j	                  |                        z   }| j                  |      | j                  | j                  | j                  | j                  |                        z   }|S r7   )
r   r   r   r   r   r   r   r   r   r   r8   s     r4   r:   zMetaFormerBlock.forwardk  s    OOA$$TZZ]3 OOAHHTZZ]+ r5   )
r=   r>   r?   r@   r   rc   r   r)   r:   rA   rB   s   @r4   r   r   A  s1      "#'!%#\Jr5   r   c            
            e Zd Zdej                  edeeddgdz  ddf
 fd	Zej                  j                  d	d       ZdefdZ xZS )
MetaFormerStager#   Frl   Nc                 b   t         |           d| _        t        |t               | _        ||k(  rt        j                         nt        ||ddd|      | _	        t        j                  t        |      D cg c]%  }t        d||||||	|
|   ||| j
                  d
|' c} | _        y c c}w )NFrq   r#   r   )r%   r&   r'   r2   )
rQ   r   r   r   r2   r{   r   r   r   rT   r   )r(   r)   grad_checkpointing
issubclassrn   rT   r*   r-   rD   
downsample
Sequentialranger   blocks)r/   in_chsout_chsdepthr   r   r   downsample_normr2   r{   dp_ratesr   r   r~   ir3   s                  r4   r)   zMetaFormerStage.__init__}  s      	"'&{I>> ,2W+<"++-,&C
 mm <&!  '6 '
#!qk#9!5]]'
 '
 &! " &!s   8*B,c                     || _         y r7   )r   )r/   enables     r4   set_grad_checkpointingz&MetaFormerStage.set_grad_checkpointing  s
    "(r5   r9   c                    | j                  |      }|j                  \  }}}}| j                  s#|j                  ||d      j	                  dd      }| j
                  r5t        j                  j                         st        | j                  |      }n| j                  |      }| j                  s$|j	                  dd      j                  ||||      }|S )Nr   r   r#   )r   rL   rT   r   r   r   rN   jitis_scriptingr   r   )r/   r9   r   r   HWs         r4   r:   zMetaFormerStage.forward  s    OOAWW
1a}}		!Q#--a3A""599+A+A+Ct{{A.AAA}}Aq!))!Q15Ar5   T)r=   r>   r?   r*   r-   rc   r   r)   rN   r   ignorer   r	   r:   rA   rB   s   @r4   r   r   {  s]     '"TAX#'!%+"Z YY) ) r5   r   c                       e Zd ZdZdddddeeddddd	d
eeedf fd	Zd Z	e
j                  j                  d"d       Ze
j                  j                  dej                  fd       Zd#dedee   fdZ	 	 	 	 	 d$de
j*                  deeeee   f      dededededeee
j*                     ee
j*                  ee
j*                     f   f   fdZ	 	 	 d%deeee   f   dedefdZd&dedefdZdefd Zdefd!Z xZS )'r   aM   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels.
        num_classes (int): Number of classes for classification head.
        global_pool: Pooling for classifier head.
        depths (list or tuple): Number of blocks at each stage.
        dims (list or tuple): Feature dimension at each stage.
        token_mixers (list, tuple or token_fcn): Token mixer for each stage.
        mlp_act: Activation layer for MLP.
        mlp_bias (boolean): Enable or disable mlp bias term.
        drop_path_rate (float): Stochastic depth rate.
        drop_rate (float): Dropout rate.
        layer_scale_init_values (list, tuple, float or None): Init value for Layer Scale.
            None means not use the layer scale. Form: https://arxiv.org/abs/2103.17239.
        res_scale_init_values (list, tuple, float or None): Init value for res Scale on residual connections.
            None means not use the res scale. From: https://arxiv.org/abs/2110.09456.
        downsample_norm (nn.Module): Norm layer used in stem and downsampling layers.
        norm_layers (list, tuple or norm_fcn): Norm layers for each stage.
        output_norm: Norm layer before classifier head.
        use_mlp_head: Use MLP classification head.
    rq   r   avgr#   r#      r#   @      i@     Frl   N)NNrW   rW   Tc                    t         |           || _        |d   | _        || _        || _        t        |      | _        t        |t        t        f      s|g}t        |t        t        f      s|g}t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }d| _        g | _        t        ||d   |      | _        g }|d   }t        j                   d|	t#        |            j%                  |      D cg c]  }|j'                          }}t)        | j                        D ]f  }|t+        |||   f||   ||   |||
||   ||   ||   |||   d
|gz  }||   }| xj                  t-        ||   d|dz   z  d|       gz  c_        h t/        j0                  | | _        |dkD  rr| j
                  r4t5        | j                  || j                  	      }| j                  | _        nFt/        j8                  | j                  |      }| j                  | _        nt/        j:                         }t/        j0                  t=        d
t?        |      fd || j                        fd|rt/        j@                  d      nt/        j:                         fd| j
                  rt/        jB                  |      nt/        j:                         fd|fg            | _"        | jG                  | jH                         y c c}w )Nr   Fr   )r2   )
r   r   r   r   r{   r   r   r   r   r2   r#   zstages.)num_chs	reductionmoduler   global_pool	pool_typer.   flattenr   r   fc)%r(   r)   r   num_featuresr   use_mlp_headlen
num_stages
isinstancelisttupler   feature_infor   stemrN   linspacesumsplittolistr   r   dictr*   r   stagesr   head_hidden_sizerv   r-   r   r   Flattenrx   headapply_init_weights)r/   in_chansr   r  depthsdimstoken_mixersr   r   drop_path_rateproj_drop_rater   layer_scale_init_valuesres_scale_init_valuesr   norm_layersoutput_normr  r~   r   prev_dimr9   r   r   finalr3   s                            r4   r)   zMetaFormer.__init__  s+   * 	& H"(f+ &4-0XF$u.6D,u6(>DOO;L+e}5&-$//9K1D%=A'>&?$//&Q#/$?%:$;doo$M!"'G&
	 7(-q.#f+(V(\(\]c(de1AHHJeet' 	cAQ Qi(O!(!!'>q'A%:1%= /&q>   F AwH$tAw!ac(U\]^\_S`"a!bb#	c& mmV, ?   1 1;$..Y(,(9(9%		$"3"3[A(,(9(9%KKMEMM+0;GH[!2!234

1"++-Hd.?.?RZZ	*R[[]S5M/
 # 	 	

4%%&U fs   %M"c                     t        |t        j                  t        j                  f      rOt	        |j
                  d       |j                  +t        j                  j                  |j                  d       y y y )Ng{Gz?)stdr   )	r  r*   r+   rv   r   weightre   init	constant_)r/   ms     r4   r%  zMetaFormer._init_weights;  sS    a"))RYY/0!((,vv!!!!&&!, " 1r5   c                 X    || _         | j                  D ]  }|j                  |        y )N)r   )r   r   r   )r/   r   stages      r4   r   z!MetaFormer.set_grad_checkpointingA  s.    "([[ 	8E(((7	8r5   returnc                 .    | j                   j                  S r7   )r#  r  )r/   s    r4   get_classifierzMetaFormer.get_classifierG  s    yy||r5   r   r  c                    || _         |Ut        |      | j                  _        |rt	        j
                  d      nt	        j                         | j                  _        |dkD  rP| j                  r#t        | j                  || j                        }n5t	        j                  | j                  |      }nt	        j                         }|| j                  _        y )Nr  r   r   r  )r   r   r#  r  r*   r"  r-   r  r  r   r  r   rv   r  )r/   r   r  r1  s       r4   reset_classifierzMetaFormer.reset_classifierK  s    &"$8;$ODII!1<

1"++-DII?   1 1;$..Y		$"3"3[AKKME		r5   r9   indicesr.   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        ||      }n ||      }||v sJ|j                  |       \ |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )r   r  r   r  rN   r   r   	enumerater   r   append)r/   r9   r?  r.   r@  rA  rB  intermediatestake_indices	max_indexr   feat_idxr9  s                r4   forward_intermediatesz MetaFormer.forward_intermediatesY  s    * Y&D(DD&"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHe&&uyy/E/E/Gua(!H<'$$Q'	(   -r5   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r  r   r>  )r/   r?  rL  rM  rH  rI  s         r4   prune_intermediate_layersz$MetaFormer.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r5   
pre_logitsc                    | j                   j                  |      }| j                   j                  |      }| j                   j                  |      }| j                   j	                  |      }|r|S | j                   j                  |      S r7   )r#  r  r.   r  r   r  )r/   r9   rQ  s      r4   forward_headzMetaFormer.forward_head  sg    II!!!$IINN1IIa IINN1q3DIILLO3r5   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r7   )r  r   rN   r   r   r   r   r8   s     r4   forward_featureszMetaFormer.forward_features  sR    IIaL""599+A+A+Ct{{A.A  AAr5   c                 J    | j                  |      }| j                  |      }|S r7   )rU  rS  r8   s     r4   r:   zMetaFormer.forward  s'    !!!$a r5   r   r7   )NFFrD  F)r   FTra   ) r=   r>   r?   r@   r   rc   r   r   r)   r%  rN   r   r   r   r*   Moduler<  r   r   strr>  r	   r   r   r   r   rK  rP  rS  rU  r:   rA   rB   s   @r4   r   r     s   6 $ $("8-)#%_'B- YY8 8
 YY		  C hsm " 8<$$',+ ||+  eCcN34+  	+ 
 +  +  !%+  
tELL!5tELL7I)I#JJ	K+ ^ ./$#	3S	>*  	4f 4$ 4&  r5   c                    d| v r| S dd l }i }d| v }|j                         }| j                         D ]  \  }}|r|j                  dd|      }|j	                  dd      }|j	                  dd	      }|j	                  d
d      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j                  dd|      }|j	                  dd      }|j	                  dd      }|j                  dd|      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j                  dd |      }|j                  d!d"|      }|j
                  ||   k7  rB|j                         ||   j                         k(  r|j                  ||   j
                        }|||<    |S )#Nzstem.conv.weightr   znetwork.0.0.mlp.fc1.weightzlayer_scale_([0-9]+)zlayer_scale\1.scalez	network.1zdownsample_layers.1z	network.3zdownsample_layers.2z	network.5zdownsample_layers.3z	network.2z	network.4z	network.6networkr   zdownsample_layers.([0-9]+)zstages.\1.downsamplezdownsample.projzdownsample.convzpatch_embed.projzpatch_embed.convz([0-9]+).([0-9]+)z\1.blocks.\2zstages.0.downsamplepatch_embedr  	post_normr.   pre_normz^headhead.fcz^normz	head.norm)re
state_dictitemssubreplacerL   numelr   )r`  modelr_  out_dictis_poolformerv1model_state_dictr   r   s           r4   checkpoint_filter_fnri    s   Z'H2j@O'')  " 1.0FJA		+'<=A		+'<=A		+'<=A		+{3A		+{3A		+{3A		)X.AFF02I1MII'):;II(*<=FF'!<II+];IImV,IIk6*IIj&)FF8Y*FF8[!,77&q))aggi;KA;N;T;T;V.V		*1-334A34 Or5   c           	          t        d t        |j                  dd            D              }|j                  d|      }t	        t
        | |ft        t        d|      d|}|S )Nc              3   &   K   | ]	  \  }}|  y wr7   r   ).0r   _s      r4   	<genexpr>z%_create_metaformer.<locals>.<genexpr>  s     \da\s   r'  r  out_indicesT)flatten_sequentialro  )pretrained_filter_fnfeature_cfg)r  rE  r   popr   r   ri  r  )variant
pretrainedr~   default_out_indicesro  re  s         r4   _create_metaformerrw    sl    \i

8\8Z.[\\**],?@K  2DkJ E Lr5   c                 0    | dddddt         t        ddd
|S )	Nr   )rq      ry  )r!   r!   rW   bicubicr^  z	stem.conv)
urlr   
input_sizer   crop_pctinterpolationmeanr3  
classifier
first_convr   )r{  r~   s     r4   _cfgr    s0    =v)%.B{  r5   zpoolformer_s12.sail_in1kztimm/g?)	hf_hub_idr}  zpoolformer_s24.sail_in1kzpoolformer_s36.sail_in1kzpoolformer_m36.sail_in1kgffffff?zpoolformer_m48.sail_in1kzpoolformerv2_s12.sail_in1k)r  zpoolformerv2_s24.sail_in1kzpoolformerv2_s36.sail_in1kzpoolformerv2_m36.sail_in1kzpoolformerv2_m48.sail_in1kzconvformer_s18.sail_in1kzhead.fc.fc2)r  r  zconvformer_s18.sail_in1k_384)rq     r  )   r  )r  r  r|  r   z!convformer_s18.sail_in22k_ft_in1kz%convformer_s18.sail_in22k_ft_in1k_384zconvformer_s18.sail_in22kiQU  )r  r  r   zconvformer_s36.sail_in1kzconvformer_s36.sail_in1k_384z!convformer_s36.sail_in22k_ft_in1kz%convformer_s36.sail_in22k_ft_in1k_384zconvformer_s36.sail_in22kzconvformer_m36.sail_in1kzconvformer_m36.sail_in1k_384z!convformer_m36.sail_in22k_ft_in1kz%convformer_m36.sail_in22k_ft_in1k_384zconvformer_m36.sail_in22kzconvformer_b36.sail_in1kzconvformer_b36.sail_in1k_384z!convformer_b36.sail_in22k_ft_in1kz%convformer_b36.sail_in22k_ft_in1k_384zconvformer_b36.sail_in22kzcaformer_s18.sail_in1kzcaformer_s18.sail_in1k_384zcaformer_s18.sail_in22k_ft_in1kz#caformer_s18.sail_in22k_ft_in1k_384zcaformer_s18.sail_in22kzcaformer_s36.sail_in1kzcaformer_s36.sail_in1k_384zcaformer_s36.sail_in22k_ft_in1kz#caformer_s36.sail_in22k_ft_in1k_384zcaformer_s36.sail_in22kzcaformer_m36.sail_in1kzcaformer_m36.sail_in1k_384zcaformer_m36.sail_in22k_ft_in1kz#caformer_m36.sail_in22k_ft_in1k_384zcaformer_m36.sail_in22kzcaformer_b36.sail_in1kzcaformer_b36.sail_in1k_384zcaformer_b36.sail_in22k_ft_in1kz#caformer_b36.sail_in22k_ft_in1k_384zcaformer_b36.sail_in22kr:  c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  r  Th㈵>F	r'  r(  r   r   r   r.  r,  r-  r  ru  r   )poolformer_s12r  r*   GELUr   rw  ru  r~   model_kwargss      r4   r  r    sO     
  $"
 
L V:VVVr5   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr"   r"   r  r"   r  Tr  Fr  ru  r   )poolformer_s24r  r  s      r4   r  r    O     
  $"
 
L V:VVVr5   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  r     r  r  Tr   Fr  ru  r   )poolformer_s36r  r  s      r4   r  r    r  r5   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  `      r     Tr   Fr  ru  r   )poolformer_m36r  r  s      r4   r  r    r  r5   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
N   r     r  r  Tr   Fr  ru  r   )poolformer_m48r  r  s      r4   r  r    r  r5   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr'  r(  r.  r  ru  r   )poolformerv2_s12r  r   rw  r  s      r4   r  r    s<      $	
 L XZX<XXr5   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  ru  r   )poolformerv2_s24r  r  s      r4   r  r    <      $	
 L XZX<XXr5   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  ru  r   )poolformerv2_s36r  r  s      r4   r  r    r  r5   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  ru  r   )poolformerv2_m36r  r  s      r4   r  r    r  r5   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  ru  r   )poolformerv2_m48r  r  s      r4   r  r     r  r5   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nrq   rq   	   rq   r  r'  r(  r)  r.  ru  r   )convformer_s18r  r   r   rw  r  s      r4   r  r    s<      %	
 L V:VVVr5   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nrq   r  r  rq   r  r  ru  r   )convformer_s36r  r  s      r4   r  r    <      %	
 L V:VVVr5   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr  r  r  r  i@  r  ru  r   )convformer_m36r  r  s      r4   r  r  !  r  r5   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr  r     r  r  r  ru  r   )convformer_b36r  r  s      r4   r  r  ,  s<     !%	
 L V:VVVr5   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r#   r  ru  r   )caformer_s18r  r   rn   r   r   rw  r  s      r4   r  r  7  sZ      w	9=&'!+.?!.CC	
 L TT|TTr5   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r#   r  ru  r   )caformer_s36r  r  s      r4   r  r  B  Z      w	9=&'!+.?!.CC	
 L TT|TTr5   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r#   r  ru  r   )caformer_m36r  r  s      r4   r  r  M  r  r5   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r#   r  ru  r   )caformer_b36r  r  s      r4   r  r  X  sZ     !w	9=&'!+.?!.CC	
 L TT|TTr5   ra   )rO  )Pr@   collectionsr   	functoolsr   typingr   r   r   r   rN   torch.nnr*   torch.nn.functional
functionalr   r	   	torch.jitr
   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__rW  r   rD   rH   rY   rc   rn   r   r   r   r   r   r   r   r   r   ri  rw  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r5   r4   <module>r     s  8 $  / /       A   * + 3 <.299 8299 </BII /
*")) 
*:ryy :06		 6xz  bll  bii  Fbii bii >7bii 7tBbii BJg gV"J  % V&!V& !	V& !V& !V& !V&" !$"9#V&$ !$"9%V&& !$"9'V&( !$"9)V&* !$"9+V&.  !"/V&4 #D ]h%P5V&: ( *";V&@ ,T ]h.PAV&F   e"5GV&N  !"OV&T #D ]h%PUV&Z ( *"[V&` ,T ]h.PaV&f   e"5gV&n  !"oV&t #D ]h%PuV&z ( *"{V&@ ,T ]h.PAV&F   e"5GV&N  !"OV&T #D ]h%PUV&Z ( *"[V&` ,T ]h.PaV&f   e"5gV&n d "oV&t !$ ]h#PuV&z &t ("{V&@ *4 ]h,PAV&F t e 5GV&N d "OV&T !$ ]h#PUV&Z &t ("[V&` *4 ]h,PaV&f t e 5gV&n d "oV&t !$ ]h#PuV&z &t ("{V&@ *4 ]h,PAV&F t e 5GV&N d "OV&T !$ ]h#PUV&Z &t ("[V&` *4 ]h,PaV&f t e 5gV& Vr W* W W W* W W W* W W W* W W W* W W YJ Y Y YJ Y Y YJ Y Y YJ Y Y YJ Y Y W* W W W* W W W* W W W* W W U
 U U U
 U U U
 U U U
 U Ur5   