
    khW              	          d Z ddlZddlmZmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lm Z m!Z! dgZ" G d dejF                        Z$ G d dejF                        Z% G d dejF                        Z& G d dejF                        Z' G d dejF                        Z( G d dejF                        Z) G d dejF                        Z* G d dejF                        Z+ G d dejF                        Z,dee-e
j\                  f   dejF                  dee-e
j\                  f   fd Z/d.d!e-d"edee-ef   fd#Z0 e  e0d$%       e0d$%       e0d$%       e0d$%      d&      Z1d/d'e-d(e2d"ede,fd)Z3e!d/d(e2d"ede,fd*       Z4e!d/d(e2d"ede,fd+       Z5e!d/d(e2d"ede,fd,       Z6e!d/d(e2d"ede,fd-       Z7y)0aj  SwiftFormer
SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications
Code: https://github.com/Amshaker/SwiftFormer
Paper: https://arxiv.org/pdf/2303.15446

@InProceedings{Shaker_2023_ICCV,
    author    = {Shaker, Abdelrahman and Maaz, Muhammad and Rasheed, Hanoona and Khan, Salman and Yang, Ming-Hsuan and Khan, Fahad Shahbaz},
    title     = {SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications},
    booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
    year      = {2023},
}
    N)AnyDictListOptionalSetTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathLinear	LayerType	to_2tupletrunc_normal_   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelSwiftFormerc                   f     e Zd Zddededef fdZdej                  dej                  fdZ	 xZ
S )	LayerScale2ddiminit_valuesinplacec                     t         |           || _        t        j                  |t        j                  |dd      z  d      | _        y )Nr   T)requires_grad)super__init__r   nn	Parametertorchonesgamma)selfr   r   r   	__class__s       S/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/swiftformer.pyr!   zLayerScale2d.__init__   s=    \\%**S!Q//tE
    xreturnc                 n    | j                   r|j                  | j                        S || j                  z  S N)r   mul_r&   r'   r+   s     r)   forwardzLayerScale2d.forward%   s(    %)\\qvvdjj!Eq4::~Er*   )h㈵>F)__name__
__module____qualname__intfloatboolr!   r$   Tensorr1   __classcell__r(   s   @r)   r   r      s?    EC Ee ET EF F%,, Fr*   r   c                        e Zd ZdZdddddej
                  fdededed	ed
edef fdZde	j                  de	j                  fdZ xZS )	Embeddingz
    Patch Embedding that is implemented by a layer of conv.
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H/stride, W/stride]
       i      r   in_chans	embed_dim
patch_sizestridepadding
norm_layerc                     t         |           t        |      }t        |      }t        |      }t        j                  |||||      | _        |r ||      | _        y t        j                         | _        y r.   )r    r!   r   r"   Conv2dprojIdentitynorm)r'   r@   rA   rB   rC   rD   rE   r(   s          r)   r!   zEmbedding.__init__/   s`     	z*
6"G$IIh	:vwO	-7Jy)	R[[]	r*   r+   r,   c                 J    | j                  |      }| j                  |      }|S r.   )rH   rJ   r0   s     r)   r1   zEmbedding.forward?   s!    IIaLIIaLr*   )r3   r4   r5   __doc__r"   BatchNorm2dr6   r   r!   r$   r9   r1   r:   r;   s   @r)   r=   r=   )   s       $&NNKK K 	K
 K K "K  %,, r*   r=   c                        e Zd ZdZdddej
                  ej                  dfdededed	ed
e	de	de
f fdZdej                  dej                  fdZ xZS )ConvEncoderz
    Implementation of ConvEncoder with 3*3 and 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Output: tensor with shape [B, C, H, W]
    @   r>           Tr   
hidden_dimkernel_size	drop_path	act_layerrE   use_layer_scalec                    t         |           t        j                  ||||dz  |      | _         ||      | _        t        j                  ||d      | _         |       | _        t        j                  ||d      | _        |dkD  rt        |      nt        j                         | _        |rt        |d      | _        y t        j                         | _        y )N   rD   groupsr   rQ   r    r!   r"   rG   dwconvrJ   pwconv1actpwconv2r   rI   rT   r   layer_scale)	r'   r   rR   rS   rT   rU   rE   rV   r(   s	           r)   r!   zConvEncoder.__init__K   s     	iiS+{a?OX[\sO	yyj!4;yyS!409B),BKKM3B<Q/r*   r+   r,   c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|| j                  |      z   }|S r.   r\   rJ   r]   r^   r_   r`   rT   )r'   r+   inputs      r)   r1   zConvEncoder.forward^   sp    KKNIIaLLLOHHQKLLOQDNN1%%r*   r3   r4   r5   rL   r"   GELUrM   r6   r7   r   r8   r!   r$   r9   r1   r:   r;   s   @r)   rO   rO   E   s     ! !#%77$&NN$(VV V 	V
 V !V "V "V&	 	%,, 	r*   rO   c                        e Zd ZdZddej
                  ej                  dfdedee   dee   de	de	d	e
f fd
Zdej                  dej                  fdZ xZS )Mlpz
    Implementation of MLP layer with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Output: tensor with shape [B, C, H, W]
    NrQ   in_featureshidden_featuresout_featuresrU   rE   dropc                    t         |           |xs |}|xs |} ||      | _        t        j                  ||d      | _         |       | _        t        j                  ||d      | _        t        j                  |      | _	        y )Nr   )
r    r!   norm1r"   rG   fc1r^   fc2Dropoutrk   )r'   rh   ri   rj   rU   rE   rk   r(   s          r)   r!   zMlp.__init__p   sq     	#2{)8[,
99[/1=;99_lA>JJt$	r*   r+   r,   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S r.   )rm   rn   r^   rk   ro   r0   s     r)   r1   zMlp.forward   sU    JJqMHHQKHHQKIIaLHHQKIIaLr*   )r3   r4   r5   rL   r"   re   rM   r6   r   r   r7   r!   r$   r9   r1   r:   r;   s   @r)   rg   rg   j   s     .2*.#%77$&NN%% &c]% #3-	%
 !% "% %$ %,, r*   rg   c                   j     e Zd ZdZd	dededef fdZdej                  dej                  fdZ xZ	S )
EfficientAdditiveAttentionz
    Efficient Additive Attention module for SwiftFormer.
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H, W]
    in_dims	token_dim	num_headsc                    t         |           |dz  | _        t        j                  |||z        | _        t        j                  |||z        | _        t        j                  t        j                  ||z  d            | _
        t        j                  ||z  ||z        | _        t        j                  ||z  |      | _        y )Ng      r   )r    r!   scale_factorr"   r   to_queryto_keyr#   r$   randnw_grH   final)r'   rt   ru   rv   r(   s       r)   r!   z#EfficientAdditiveAttention.__init__   s    %-		'9y+@AiiY)>?<<I	,A1 EFIIi)3Y5JK	YYy94i@
r*   r+   r,   c                 2   |j                   \  }}}}|j                  d      j                  ddd      }t        j                  | j                  |      d      }t        j                  | j                  |      d      }t        j                  || j                  z  | j                  z  d      }t        j                  ||z  dd      }| j                  ||z        |z   }	| j                  |	      j                  ddd      j                  |d||      }	|	S )NrX   r   r   r   T)r   keepdim)shapeflattenpermuteF	normalizery   rz   r|   rx   r$   sumrH   r}   reshape)
r'   r+   B_HWquerykeyattnouts
             r)   r1   z"EfficientAdditiveAttention.forward   s    WW
1aIIaL  Aq)DMM!,"5kk$++a.b1{{5488+d.?.??QGyy1d;iis
#e+jjo%%aA.66q"aC
r*   )      r   )
r3   r4   r5   rL   r6   r!   r$   r9   r1   r:   r;   s   @r)   rs   rs      sA    
	A 	Ac 	AC 	A %,, r*   rs   c                        e Zd ZdZdddej
                  ej                  fdededede	d	e
d
e
f fdZdej                  dej                  fdZ xZS )LocalRepresentationz
    Local Representation module for SwiftFormer that is implemented by 3*3 depth-wise and point-wise convolutions.
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H, W]
    r>   rQ   Tr   rS   rT   rV   rU   rE   c                    t         |           t        j                  ||||dz  |      | _         ||      | _        t        j                  ||d      | _         |       | _        t        j                  ||d      | _        |dkD  rt        |      nt        j                         | _        |rt        |d      | _        y t        j                         | _        y )NrX   rY   r   )rS   rQ   r[   )r'   r   rS   rT   rV   rU   rE   r(   s          r)   r!   zLocalRepresentation.__init__   s     	iiS+{a?OX[\sO	yycq9;yycq909B),BKKM3B<Q/r*   r+   r,   c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|| j                  |      z   }|S r.   rb   )r'   r+   skips      r)   r1   zLocalRepresentation.forward   sp    KKNIIaLLLOHHQKLLOQ4>>!$$r*   )r3   r4   r5   rL   r"   re   rM   r6   r7   r8   r   r!   r$   r9   r1   r:   r;   s   @r)   r   r      s      !!$(#%77$&NNVV V 	V
 "V !V "V$	 	%,, 	r*   r   c                        e Zd ZdZdddej
                  ej                  ddfdededed	ed
e	de	de
def fdZdej                  dej                  fdZ xZS )Blockz
    SwiftFormer Encoder Block for SwiftFormer. It consists of :
    (1) Local representation module, (2) EfficientAdditiveAttention, and (3) MLP block.
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H, W]
          @rQ   Tr2   r   	mlp_ratio	drop_raterT   rU   rE   rV   layer_scale_init_valuec	                    t         	|           t        ||||      | _        t	        ||      | _        t        |t        ||z        |||      | _        |dkD  rt        |      nt        j                         | _        |rt        ||      nt        j                         | _        |rt        ||      | _        y t        j                         | _        y )N)r   rV   rU   rE   )rt   ru   )rh   ri   rU   rE   rk   rQ   )r    r!   r   local_representationrs   r   rg   r6   linearr   r"   rI   rT   r   layer_scale_1layer_scale_2)
r'   r   r   r   rT   rU   rE   rV   r   r(   s
            r)   r!   zBlock.__init__   s     	$7+!	%
! /scJ	i0!
 1:B),BKKM *#/EF$&KKM 	  *#/EF$&KKM 	r*   r+   r,   c                     | j                  |      }|| j                  | j                  | j                  |                  z   }|| j                  | j	                  | j                  |                  z   }|S r.   )r   rT   r   r   r   r   r0   s     r)   r1   zBlock.forward   sc    %%a(t11$))A,?@@t11$++a.ABBr*   rd   r;   s   @r)   r   r      s      "!!#%77$&NN$(,022 2 	2
 2 !2 "2 "2 %*2@ %,, r*   r   c                        e Zd ZdZdej
                  ej                  dddddfdeded	ee   d
e	de
de
de	de	dede	dee
   f fdZdej                  dej                  fdZ xZS )Stagez
    Implementation of each SwiftFormer stages. Here, SwiftFormerEncoder used as the last block in all stages, while ConvEncoder used in the rest of the blocks.
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H, W]
    r   rQ   Tr2   Nr   indexlayersr   rU   rE   r   drop_path_raterV   r   
downsamplec                    t         |           d| _        ||nt        j                         | _        g }t        ||         D ]  }||t        |d |       z   z  t        |      dz
  z  }||   |z
  dk  r#|j                  t        |||||||	|
             T|j                  t        |t        ||z        d||||	              t        j                  | | _        y )NFr   )r   r   rT   rU   rE   rV   r   r>   )r   rR   rS   rT   rU   rE   rV   )r    r!   grad_checkpointingr"   rI   r   ranger   appendr   rO   r6   
Sequentialblocks)r'   r   r   r   r   rU   rE   r   r   rV   r   r   r   	block_idx	block_dprr(   s                  r)   r!   zStage.__init__  s     	"'(2(>*BKKMve}- 	I&)c&%.6I*IJcRXk\]o^Ie}y(A-e'''')$3+A	 	 k"9s?3 !'')$3 	. mmV,r*   r+   r,   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r.   )r   r   r$   jitis_scriptingr   r   r0   s     r)   r1   zStage.forward0  sS    OOA""599+A+A+Ct{{A.A  AAr*   )r3   r4   r5   rL   r"   re   rM   r6   r   r7   r   r8   r   r!   r$   r9   r1   r:   r;   s   @r)   r   r      s      "#%77$&NN!$&$(,0.2*-*- *- I	*-
 *- !*- "*- *- "*- "*- %**- !+*-X %,, r*   r   c            !           e Zd Zg dg ddg dej                  ddddd	d	d
ddddfdee   dee   dedee   dededededede	de	dede	de
dedef  fdZd Zej                  j                  d efd!       Zej                  j                  d7d"ed ee
ef   fd#       Zej                  j                  d8d$efd%       Zej                  j                  d eej.                  ej.                  f   fd&       Zd9dedee
   fd'Zej                  j                  d8d$efd(       Z	 	 	 	 	 d:d)ej8                  d*eeeee   f      d+ed,ed-e
d.ed eeej8                     eej8                  eej8                     f   f   fd/Z	 	 	 d;d*eeee   f   d0ed1efd2Zd)ej8                  d ej8                  fd3Z d7d)ej8                  d4efd5Z!d)ej8                  fd6Z" xZ#S )<r   r>   r>         0   8   p      r   )FTTTr>   rX   r     rQ   Tr2   avg    r   
embed_dims
mlp_ratiosdownsamplesrU   down_patch_sizedown_stridedown_padnum_classesr   r   rV   r   global_pooloutput_strider@   c                    t         |           |dk(  sJ |	| _        || _        g | _        t        j                  t        j                  ||d   dz  ddd      t        j                  |d   dz        t        j                         t        j                  |d   dz  |d   ddd      t        j                  |d         t        j                               | _
        |d   }g }t        t        |            D ]  }||   rt        |||   |||      nt        j                         }t        ||   |||||
||||
      }||   }|j!                  |       | xj                  t#        ||   d|dz   z  d| 	      gz  c_         t        j                  | | _        |d
   x| _        x| _        }t        j                  |      | _        t        j,                  |
      | _        |	dkD  rt1        ||	      nt        j                         | _        |	dkD  rt1        ||	      nt        j                         | _        d| _        | j9                          y )Nr   r   rX   r>   r   )r@   rA   rB   rC   rD   )
r   r   r   r   rU   r   r   rV   r   r   stages.)num_chs	reductionmoduler   F)r    r!   r   r   feature_infor"   r   rG   rM   ReLUstemr   lenr=   rI   r   r   dictstagesnum_featureshead_hidden_sizerJ   rp   	head_dropr   head	head_distdistilled_training_initialize_weights)r'   r   r   r   r   rU   r   r   r   r   r   r   rV   r   r   r   r@   kwargsprev_dimr   ir   stageout_chsr(   s                           r)   r!   zSwiftFormer.__init__:  s.   ( 	"""&&MMIIh
1 2Aq!<NN:a=A-.GGIIIjmq(*Q-AqANN:a=)GGI
	 a=s6{# 	iA Q #!$Q-*"  &([[]  qM$##- /'=%E "!}HMM% $z!}AaC[bcdbeYf"g!hh-	i. mmV, @J"~MMT2WNN7+	I.4?!OF7K0	9Dq5bkkm"'  "r*   c                    | j                         D ]  \  }}t        |t        j                        rOt	        |j
                  d       |j                  Dt        j                  j                  |j                  d       ot        |t        j                        st	        |j
                  d       |j                  t        j                  j                  |j                  d        y )Ng{Gz?)stdr   )
named_modules
isinstancer"   r   r   weightbiasinit	constant_rG   )r'   namems      r)   r   zSwiftFormer._initialize_weights  s    ))+ 	1GD!!RYY'ahhC066%GG%%affa0Aryy)ahhC066%GG%%affa0	1r*   r,   c                     t               S r.   )setr'   s    r)   no_weight_decayzSwiftFormer.no_weight_decay  s	    ur*   coarsec                 ,    t        d|rdng d      }|S )Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)z^norm)i )r   r   )r   )r'   r   matchers      r)   group_matcherzSwiftFormer.group_matcher  s!    (.$ 5
 r*   enablec                 4    | j                   D ]	  }||_         y r.   )r   r   )r'   r   ss      r)   set_grad_checkpointingz"SwiftFormer.set_grad_checkpointing  s     	*A#)A 	*r*   c                 2    | j                   | j                  fS r.   r   r   r   s    r)   get_classifierzSwiftFormer.get_classifier  s    yy$..((r*   c                     || _         ||| _        |dkD  rt        | j                  |      nt	        j
                         | _        |dkD  rt        | j                  |      | _        y t	        j
                         | _        y )Nr   )r   r   r   r   r"   rI   r   r   )r'   r   r   s      r)   reset_classifierzSwiftFormer.reset_classifier  si    &"*D>IAoF4,,k:SUS^S^S`	CNQR? 1 1;?XZXcXcXer*   c                     || _         y r.   )r   )r'   r   s     r)   set_distilled_trainingz"SwiftFormer.set_distilled_training  s
    "(r*   r+   indicesrJ   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	t        | j                        dz
  }
| j                  |      }t        j
                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]>  \  }} ||      }||v s|r||
k(  r| j                  |      }n|}|j                  |       @ |r|S |
k(  r| j                  |      }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r   N)
r   r   r   r   r$   r   r   	enumeraterJ   r   )r'   r+   r   rJ   r   r   r  intermediatestake_indices	max_indexlast_idxr   feat_idxr   x_inters                  r)   forward_intermediatesz!SwiftFormer.forward_intermediates  s   * Y&D(DD&"6s4;;7G"Qit{{#a' IIaL99!!#:[[F[[)a-0F(0 	.OHeaA<'H0"iilGG$$W-	.   x		!A-r*   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r   r"   rI   rJ   r   )r'   r   r  r  r  r  s         r)   prune_intermediate_layersz%SwiftFormer.prune_intermediate_layers  s]     #7s4;;7G"Qikk.9q=1DI!!!R(r*   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r.   )r   r   rJ   r0   s     r)   forward_featureszSwiftFormer.forward_features  s.    IIaLKKNIIaLr*   
pre_logitsc                 6   | j                   dk(  r|j                  d      }| j                  |      }|r|S | j                  |      | j	                  |      }}| j
                  r.| j                  r"t        j                  j                         s||fS ||z   dz  S )Nr   )rX   r>   r   rX   )
r   meanr   r   r   r   trainingr$   r   r   )r'   r+   r  x_dists       r)   forward_headzSwiftFormer.forward_head  s    u$6"ANN1HIIaL$.."36""t}}UYY=S=S=Uf9 J!##r*   c                 J    | j                  |      }| j                  |      }|S r.   )r  r  r0   s     r)   r1   zSwiftFormer.forward  s'    !!!$a r*   F)Tr.   )NFFr  F)r   FT)$r3   r4   r5   r"   re   r   r6   r8   r   r7   strr!   r   r$   r   ignorer   r   r   r   r   r   r   Moduler   r   r   r   r9   r	   r  r  r  r  r1   r:   r;   s   @r)   r   r   9  s	    !-$6&?#%77#$ #!$&$(,0$!##F#IF# S	F# 	F#
 dF# !F# !F# F# F# F# F# "F# "F# %*F# F#  !F#" #F#P	1 YY   YY	D 	T#s(^ 	 	 YY*T * * YY)bii&: ; ) )fC fhsm f YY)T ) ) 8<$$',0 ||0  eCcN340  	0 
 0  0  !%0  
tELL!5tELL7I)I#JJ	K0 h ./$#	3S	>*  	 %,, 5<< $ell $ $ r*   
state_dictmodelr,   c                    | j                  d|       } d| v r| S i }| j                         D ]  \  }}|j                  dd      }|j                  dd      }|j                  dd      }|j                  d	d
      }|j                  dd      }t        j                  dd|      }t        j
                  d|      }|rLt        |j                  d            |j                  d      }}|dz  }|dz  dk(  r	d| d| }nd|dz    d| }|||<    |S )Nr  zstem.0.weightzpatch_embed.zstem.z
dist_head.z
head_dist.z
attn.Proj.z
attn.proj.z.layer_scale_1z.layer_scale_1.gammaz.layer_scale_2z.layer_scale_2.gammaz\.layer_scale(?=$|\.)z.layer_scale.gammaz^network\.(\d+)\.(.*)r   rX   r   r   z.blocks.z.downsample.)getitemsreplaceresubmatchr6   group)	r  r  out_dictkvr   n_idxrest	stage_idxs	            r)   checkpoint_filter_fnr.    s&   4J*$H  " 1IIng.IIlL1IIlL1II&(>?II&(>?FF+-A1EHH-q1aggaj/1771:4E
IqyA~i[7ik],tf=!" Or*   urlr   c                 8    | ddd dddt         t        dddd	d
d|S )Nr   )r>      r1  Tgffffff?bicubiczstem.0r   zarXiv:2303.15446zdSwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applicationsz'https://github.com/Amshaker/SwiftFormer)r/  r   
input_size	pool_sizefixed_input_sizecrop_pctinterpolationr  r   
first_conv
classifier	paper_ids
paper_name
origin_urlr
   )r/  r   s     r)   _cfgr=  '  s=    =tae)%.B.C'|?
 
 
r*   ztimm/)	hf_hub_id)zswiftformer_xs.dist_in1kzswiftformer_s.dist_in1kzswiftformer_l1.dist_in1kzswiftformer_l3.dist_in1kvariant
pretrainedc                 N    t        t        | |ft        t        dd      d|}|S )N)r   r   rX   r>   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r.  r   )r?  r@  r   r  s       r)   _create_swiftformerrF  E  s6     Wj1\dK 	E Lr*   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   r   r   r   r@  )swiftformer_xsr   rF  r@  r   
model_argss      r)   rI  rI  O  s-    \6HIJeJe$zJd]cJdeer*   c           	      R    t        g dg d      }t        dd| it        |fi |S )N)r>   r>   	   r   )r   rP      r1  rH  r@  )swiftformer_srJ  rK  s      r)   rP  rP  U  s-    \6HIJd:djIc\bIcddr*   c           	      R    t        g dg d      }t        dd| it        |fi |S )N)r   r>   
      )r   `      i  rH  r@  )swiftformer_l1rJ  rK  s      r)   rV  rV  Z  s-    ]7IJJeJe$zJd]cJdeer*   c           	      R    t        g dg d      }t        dd| it        |fi |S )N)r   r      r   )rP      i@  r   rH  r@  )swiftformer_l3rJ  rK  s      r)   rZ  rZ  `  s-    ]7JKJeJe$zJd]cJdeer*   )r  r  )8rL   r$  typingr   r   r   r   r   r   r	   r$   torch.nnr"   torch.nn.functional
functionalr   	timm.datar   r   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r   r=   rO   rg   rs   r   r   r   r   r  r9   r.  r=  default_cfgsr8   rF  rI  rP  rV  rZ   r*   r)   <module>rh     sY   
 ? ? ?     A M M * + ' </F299 F		 8"")) "J")) D @!")) !H+BII +\8BII 8vQ")) QhT#u||*;%< RYY SWX[]b]i]iXiSj 4c # $sCx.  % $!  $  !%! !%!&   $ # R]  ft fs f{ f f
 ed ec ek e e ft fs f{ f f
 ft fs f{ f fr*   