
    kh_f                        d Z dgZddlZddlmZ ddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z&  G d dej                  jN                        Z( G d dejR                        Z* G d dejR                        Z+ G d dejR                        Z, G d dejR                        Z- G d dejR                        Z. G d dej                  jR                        Z/ G d dejR                        Z0 e e0        G d dejR                        Z1 G d  dejR                        Z2d! Z3d6d"Z4 e& e4d#d$%       e4d#&       e4d#&       e4d#d$%       e4d#&       e4d#&       e4d#d$%       e4d#&       e4d#&       e4d#d'd(d)*       e4d#d+d,d)d-.      d/      Z5d7d0Z6e%d7d1       Z7e%d7d2       Z8e%d7d3       Z9e%d7d4       Z:e%d7d5       Z;y)8z TinyViT

Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers`
    - https://arxiv.org/abs/2207.10666

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT
TinyVit    N)partial)DictListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)LayerNorm2dNormMlpClassifierHeadDropPathtrunc_normal_resize_rel_pos_bias_table_levituse_fused_attn   )build_model_with_cfg)feature_take_indices)register_notrace_module)
checkpointcheckpoint_seq)register_modelgenerate_default_cfgsc                   N     e Zd Zd fd	Z ej
                         d        Z xZS )ConvNormc	           
         t         	|           t        j                  |||||||d      | _        t        j
                  |      | _        t        j                  j                  j                  | j                  j                  |       t        j                  j                  j                  | j                  j                  d       y )NF)biasr   )super__init__nnConv2dconvBatchNorm2dbntorchinit	constant_weightr   )
selfin_chsout_chsksstridepaddilationgroupsbn_weight_init	__class__s
            P/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/tiny_vit.pyr    zConvNorm.__init__   s}    IIfgr63&W\]	..)?a0    c           	      B   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  j                  |j                  d      | j                   j                  z  |j                  d      |j                  dd  | j                   j                  | j                   j                  | j                   j                  | j                   j                        }|j                  j                   j#                  |       |j
                  j                   j#                  |       |S )Ng      ?r   r      )r.   paddingr0   r1   )r#   r%   r)   running_varepsr   running_meanr&   r!   r"   sizer1   shaper.   r8   r0   datacopy_)r*   cr%   wbms         r4   fusezConvNorm.fuse%   s5   		4772II"&&0S88HHqD$,--GGboo		1^^bff$,- -HHOOFF1I		(((!&&)QWWQR[99##TYY->->I[I[dhdmdmdtdt  v 	
A	!r5   )r   r   r   r   r   r   )__name__
__module____qualname__r    r&   no_gradrD   __classcell__r3   s   @r4   r   r      s$    1 U]]_ r5   r   c                   $     e Zd Z fdZd Z xZS )
PatchEmbedc                     t         |           d| _        t        ||dz  ddd      | _         |       | _        t        |dz  |ddd      | _        y )N   r7      r   )r   r    r.   r   conv1actconv2)r*   r+   r,   	act_layerr3   s       r4   r    zPatchEmbed.__init__5   sN    fglAq!<
;glGQ1=
r5   c                 l    | j                  |      }| j                  |      }| j                  |      }|S N)rP   rQ   rR   r*   xs     r4   forwardzPatchEmbed.forward<   s.    JJqMHHQKJJqMr5   rE   rF   rG   r    rX   rI   rJ   s   @r4   rL   rL   4   s    >r5   rL   c                   $     e Zd Z fdZd Z xZS )MBConvc                 ^   t         |           t        ||z        }t        ||d      | _         |       | _        t        ||ddd|      | _         |       | _        t        ||dd      | _         |       | _	        |dkD  rt        |      | _        y t        j                         | _        y )Nr   )r-   rO   r-   r.   r/   r1           )r-   r2   )r   r    intr   rP   act1rR   act2conv3act3r   r!   Identity	drop_path)r*   r+   r,   expand_ratiorS   re   mid_chsr3   s          r4   r    zMBConv.__init__D   s    f|+,fg!4
K	gw1QAgV
K	gw1SI
K	09B),BKKMr5   c                    |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }||z  }| j                  |      }|S rU   )rP   r`   rR   ra   rb   re   rc   )r*   rW   shortcuts      r4   rX   zMBConv.forwardO   sq    JJqMIIaLJJqMIIaLJJqMNN1	XIIaLr5   rY   rJ   s   @r4   r[   r[   C   s    	R
r5   r[   c                   $     e Zd Z fdZd Z xZS )PatchMergingc                     t         |           t        ||ddd      | _         |       | _        t        ||ddd|      | _         |       | _        t        ||ddd      | _        y )Nr   r   rO   r7   )r1   )r   r    r   rP   r`   rR   ra   rb   )r*   dimout_dimrS   r3   s       r4   r    zPatchMerging.__init__]   s^    c7Aq!4
K	gw1aH
K	gw1a8
r5   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S rU   )rP   r`   rR   ra   rb   rV   s     r4   rX   zPatchMerging.forwarde   sH    JJqMIIaLJJqMIIaLJJqMr5   rY   rJ   s   @r4   rk   rk   \   s    9r5   rk   c                   *     e Zd Z	 	 d fd	Zd Z xZS )	ConvLayerc                     t         |           || _        || _        t	        j
                  t        |      D cg c]&  }t        ||||t        |t              r||   n|      ( c} | _
        y c c}w rU   )r   r    rm   depthr!   
Sequentialranger[   
isinstancelistblocks)r*   rm   rs   rS   re   conv_expand_ratioir3   s          r4   r    zConvLayer.__init__o   sr     	
mm
 5\&

 	 S+Y *9d ;	!&
  &
s   +A/c                 (    | j                  |      }|S rU   )rx   rV   s     r4   rX   zConvLayer.forward   s    KKNr5   )r^         @rY   rJ   s   @r4   rq   rq   n   s      &r5   rq   c                   X     e Zd Zddej                  ej
                  df fd	Zd Z xZS )NormMlpNr^   c                 @   t         |           |xs |}|xs |} ||      | _        t        j                  ||      | _         |       | _        t        j                  |      | _        t        j                  ||      | _	        t        j                  |      | _
        y rU   )r   r    normr!   Linearfc1rQ   Dropoutdrop1fc2drop2)r*   in_featureshidden_featuresout_features
norm_layerrS   dropr3   s          r4   r    zNormMlp.__init__   s}     	#2{)8[{+	99[/:;ZZ%
99_l;ZZ%
r5   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S rU   )r   r   rQ   r   r   r   rV   s     r4   rX   zNormMlp.forward   sU    IIaLHHQKHHQKJJqMHHQKJJqMr5   )	rE   rF   rG   r!   	LayerNormGELUr    rX   rI   rJ   s   @r4   r~   r~      s&     !||gg&&r5   r~   c                        e Zd ZU ej                  j
                  e   ed<   ee	ej                  f   ed<   	 	 	 d	 fd	Z ej                         d
 fd	       Zdej                  dej                  fdZd Z xZS )	Attention
fused_attnattention_bias_cachec           	      6   t         |           t        |t              rt	        |      dk(  sJ || _        |dz  | _        || _        t        ||z        | _	        | j                  |z  | _
        || _        || _        t               | _        t        j                   |      | _        t        j$                  ||| j                  d|z  z   z        | _        t        j$                  | j                  |      | _        t+        t-        j.                  t1        |d         t1        |d                     }t	        |      }i }g }	|D ]W  }
|D ]P  }t3        |
d   |d   z
        t3        |
d   |d   z
        f}||vrt	        |      ||<   |	j5                  ||          R Y t6        j                  j9                  t7        j:                  |t	        |                  | _        | j?                  dt7        j@                  |	      jC                  ||      d       i | _"        y )Nr7   g      r   r   attention_bias_idxsF)
persistent)#r   r    rv   tuplelen	num_headsscalekey_dimr_   val_dimrn   
attn_ratio
resolutionr   r   r!   r   r   r   qkvprojrw   	itertoolsproductru   absappendr&   	Parameterzerosattention_biasesregister_buffer
LongTensorviewr   )r*   rm   r   r   r   r   pointsNattention_offsetsidxsp1p2offsetr3   s                r4   r    zAttention.__init__   s    	*e,ZA1EEE"_
:/0||i/$$(*LL%	99S)t||a'k/I"JKIIdllC0	i''jm(<eJqM>RSTK 	7B 7bebem,c"Q%"Q%-.@A!22034E0F%f--f56	7	7 !& 2 25;;y#N_J`3a b2E4D4DT4J4O4OPQST4Ubgh$&!r5   c                 R    t         |   |       |r| j                  ri | _        y y y rU   )r   trainr   )r*   moder3   s     r4   r   zAttention.train   s)    dD--(*D% .4r5   devicereturnc                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rU   )r&   jit
is_tracingtrainingr   r   strr   )r*   r   
device_keys      r4   get_attention_biaseszAttention.get_attention_biases   s    99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r5   c                    | j                  |j                        }|j                  \  }}}| j                  |      }| j	                  |      }|j                  ||| j                  d      j                  | j                  | j                  | j                  gd      \  }}}	|j                  dddd      }|j                  dddd      }|	j                  dddd      }	| j                  rt        j                  |||	|      }n@|| j                  z  }||j                  dd      z  }
|
|z   }
|
j!                  d      }
|
|	z  }|j                  dd      j#                  ||| j$                        }| j'                  |      }|S )	NrO   )rm   r   r7   r   )	attn_mask)r   r   r=   r   r   r   r   splitr   r   permuter   Fscaled_dot_product_attentionr   	transposesoftmaxreshapern   r   )r*   rW   	attn_biasBr   _r   qkvattns              r4   rX   zAttention.forward   sW   --ahh7	''1aIIaLhhqk((1a4::DLL$,,X\XdXd;ekl:m1aIIaAq!IIaAq!IIaAq!??..q!Q)LADJJAq{{2r**D)#D<<B<'DqAKK1%%aDLL9IIaLr5   )   rN   )   r   T)rE   rF   rG   r&   r   Finalbool__annotations__r   r   Tensorr    rH   r   r   r   rX   rI   rJ   s   @r4   r   r      sw    		%%sELL011 #'J U]]_+ +
95<< 9ELL 9r5   r   c                   V     e Zd ZdZdddddej
                  f fd	Zd Zdefd	Z	 xZ
S )
TinyVitBlocka5   TinyViT Block.

    Args:
        dim (int): Number of input channels.
        num_heads (int): Number of attention heads.
        window_size (int): Window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float, optional): Stochastic depth rate. Default: 0.0
        local_conv_size (int): the kernel size of the convolution between
                               Attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
       r|   r^   rO   c	                    t         |           || _        || _        |dkD  sJ d       || _        || _        ||z  dk(  sJ d       ||z  }	||f}
t        ||	|d|
      | _        |dkD  rt        |      nt        j                         | _        t        |t        ||z        ||      | _        |dkD  rt        |      nt        j                         | _        |dz  }t!        |||d||	      | _        y )
Nr   z"window_size must be greater than 0z"dim must be divisible by num_headsr   )r   r   r^   )r   r   rS   r   r7   r]   )r   r    rm   r   window_size	mlp_ratior   r   r   r!   rd   
drop_path1r~   r_   mlp
drop_path2r   
local_conv)r*   rm   r   r   r   r   re   local_conv_sizerS   head_dimwindow_resolutionr/   r3   s               r4   r    zTinyVitBlock.__init__  s     	"QD DD&"Y!#I%II#)#(+6c8Y1Qbc	1:R(9-R[[] i0	
 2;R(9-R[[]""3s[^_r5   c           	         |j                   \  }}}}||z  }|}|| j                  k(  rI|| j                  k(  r:|j                  |||      }| j                  |      }|j	                  ||||      }n| j                  || j                  z  z
  | j                  z  }| j                  || j                  z  z
  | j                  z  }	|dkD  xs |	dkD  }
|
rt        j                  |ddd|	d|f      }||z   ||	z   }}|| j                  z  }|| j                  z  }|j	                  ||| j                  || j                  |      j                  dd      j                  ||z  |z  | j                  | j                  z  |      }| j                  |      }|j	                  |||| j                  | j                  |      j                  dd      j                  ||||      }|
r|d d d |d |f   j                         }|| j                  |      z   }|j                  dddd      }| j                  |      }|j                  |||      j                  dd      }|| j                  | j                  |            z   }|j	                  ||||      S Nr   r7   rO   r   )r=   r   r   r   r   r   r/   r   
contiguousr   r   r   r   r   )r*   rW   r   HWCLri   pad_bpad_rr8   pHpWnHnWs                  r4   rX   zTinyVitBlock.forward+  s~   WW
1aE   Q$*:*:%:		!Q"A		!Aq!Q"A%%D,<,<(<<@P@PPE%%D,<,<(<<@P@PPEai,519GEE!aAua78 YE	Bt'''Bt'''Bq"d..D4D4DaHRRSTVWX``BT--0@0@@!A 		!A q"b$"2"2D4D4DaHRRSTVWX``abdfhjlmnAa!RaRiL++-tq))IIaAq!OOAIIaA((A.,,vvaAq!!r5   r   c                 n    d| j                    d| j                   d| j                   d| j                   S )Ndim=z, num_heads=z, window_size=z, mlp_ratio=)rm   r   r   r   r*   s    r4   
extra_reprzTinyVitBlock.extra_reprS  s@    dhhZ|DNN+; <"../|DNN;KM 	Mr5   rE   rF   rG   __doc__r!   r   r    rX   r   r   rI   rJ   s   @r4   r   r      s;    $ gg#`J&"PMC Mr5   r   c                   V     e Zd ZdZdddddej
                  f fd	Zd Zdefd	Z	 xZ
S )
TinyVitStagea   A basic TinyViT layer for one stage.

    Args:
        dim (int): Number of input channels.
        out_dim: the output dimension of the layer
        depth (int): Number of blocks.
        num_heads (int): Number of attention heads.
        window_size (int): Local window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
        local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
    r|   r^   NrO   c                 V   t         |           || _        || _        |	 |	|||      | _        n t        j                         | _        ||k(  sJ t        j                  t        |      D cg c]*  }t        |||||t        |t              r||   n||
|      , c} | _        y c c}w )N)rm   rn   rS   )rm   r   r   r   r   re   r   rS   )r   r    rs   rn   
downsampler!   rd   rt   ru   r   rv   rw   rx   )r*   rm   rn   rs   r   r   r   r   re   r   r   rS   rz   r3   s                r4   r    zTinyVitStage.__init__l  s     	
 !(#DO !kkmDO'>!> mm 5\&#  #'#*4Y*E)A,9 /#	&# $ &#s   -/B&c                     | j                  |      }|j                  dddd      }| j                  |      }|j                  dddd      }|S r   )r   r   rx   rV   s     r4   rX   zTinyVitStage.forward  sJ    OOAIIaAq!KKNIIaAq!r5   r   c                 :    d| j                    d| j                   S )Nr   z, depth=)rn   rs   r   s    r4   r   zTinyVitStage.extra_repr  s    dll^8DJJ<88r5   r   rJ   s   @r4   r   r   [  s7    . gg*$X9C 9r5   r   c                       e Zd Zddddddddd	d
dddej                  f fd	Zd Zej                  j                  d        Z
ej                  j                  d        Zej                  j                  d%d       Zej                  j                  d&d       Zej                  j                  dej                  fd       Zd'dedee   fdZ	 	 	 	 	 d(dej(                  deeeee   f      dededededeeej(                     eej(                  eej(                     f   f   fdZ	 	 	 d)deeee   f   dedefd Zd! Zd%d"efd#Zd$ Z xZS )*r   rO     avg)`        i   r7   r7      r7   )rO   r        r   r   r   r   r|   r^   皙?Fc                    t         |           || _        || _        t	        |      | _        || _        || _        t        ||d   |      | _	        t        j                  d|
t        |            D cg c]  }|j                          }}t        j                         | _        | j                  j"                  }|d   }g | _        t'        | j
                        D ]  }|dk(  rt)        |||   ||d ||    |      }n_||   }|t        |d |       t        |d |dz           }
t+        ||dz
     |||   ||   ||   | j                  |	||
t,        |      }|}|dz  }| j                   j/                  |       | xj$                  t1        ||d|       gz  c_         |d	   x| _        | _        t7        t8        d
      }t;        | j2                  |||      | _        | j?                  | j@                         y c c}w )Nr   )r+   r,   rS   )rm   rs   rS   re   ry   r   )rm   rn   rs   r   r   r   r   r   re   r   rS   r7   zstages.)num_chs	reductionmoduler   gh㈵>)r:   )	pool_typer   )!r   r    num_classesdepthsr   
num_stagesr   grad_checkpointingrL   patch_embedr&   linspacesumitemr!   rt   stagesr.   feature_inforu   rq   r   rk   r   dictnum_featureshead_hidden_sizer   r   r   headapply_init_weights)r*   in_chansr  global_pool
embed_dimsr  r   window_sizesr   	drop_ratedrop_path_rateuse_checkpointmbconv_expand_ratior   rS   rW   dprr.   prev_dim	stage_idxstagern   norm_layer_cfr3   s                          r4   r    zTinyVit.__init__  s   " 	&f+""0%qM
 "'>3v;!OPAqvvxPP mmo!!((a=t/ 	jIA~!  +'!"46)#45&9 %Y/!$S
);%<SUVAW=X!Y$"9q=1# +'	2 ,Y 7"nn"$3,+' #!KKu%$x6T[\e[fRg"h!ii9	j> 5?rNBD16)!$	
	 	

4%%&c Qs   4G<c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y y )Ng{Gz?)stdr   )rv   r!   r   r   r)   r   r'   r(   )r*   rC   s     r4   r  zTinyVit._init_weights  sZ    a#!((,!RYY'AFF,>!!!&&!, -?' $r5   c                     dhS Nr    r   s    r4   no_weight_decay_keywordsz TinyVit.no_weight_decay_keywords  s    "##r5   c                 n    | j                         j                         D ch c]	  }d|v s| c}S c c}w r-  )
state_dictkeysrV   s     r4   no_weight_decayzTinyVit.no_weight_decay  s.    ??,113Oa7IQ7NOOOs   	22c                 ,    t        d|rdnddg      }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)stemrx   )r  )r*   coarsematchers      r4   group_matcherzTinyVit.group_matcher  s'     (.$455
 r5   c                     || _         y rU   )r  )r*   enables     r4   set_grad_checkpointingzTinyVit.set_grad_checkpointing  s
    "(r5   r   c                 .    | j                   j                  S rU   )r  fcr   s    r4   get_classifierzTinyVit.get_classifier  s    yy||r5   r  r  c                 L    || _         | j                  j                  ||       y )N)r  )r  r  reset)r*   r  r  s      r4   reset_classifierzTinyVit.reset_classifier  s    &		{;r5   rW   indicesr   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        ||      }n ||      }||v sJ|j                  |       \ |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )r   r   r  r  r&   r   is_scripting	enumerater  r   r   )r*   rW   rB  r   rC  rD  rE  intermediatestake_indices	max_indexr  feat_idxr(  s                r4   forward_intermediateszTinyVit.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qi Q99!!#:[[F[[)a-0F(0 	(OHe&&uyy/E/E/Gua(!H<'$$Q'	(   -r5   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r  rA  )r*   rB  rO  rP  rK  rL  s         r4   prune_intermediate_layersz!TinyVit.prune_intermediate_layersI  sM     #7s4;;7G"Qikk.9q=1!!!R(r5   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S rU   )r  r  r&   r   rH  r   r  rV   s     r4   forward_featureszTinyVit.forward_featuresW  sU    Q""599+A+A+Ct{{A.A  AAr5   
pre_logitsc                 V    |r| j                  ||      }|S | j                  |      }|S )N)rV  )r  )r*   rW   rV  s      r4   forward_headzTinyVit.forward_head_  s1    3=DIIaJI/ DH99Q<r5   c                 J    | j                  |      }| j                  |      }|S rU   )rU  rX  rV   s     r4   rX   zTinyVit.forwardc  s'    !!!$a r5   Fr   rU   )NFFrG  F)r   FT)rE   rF   rG   r!   r   r    r  r&   r   ignorer/  r3  r8  r;  Moduler>  r_   r   r   rA  r   r	   r   r   r   rN  rS  rU  rX  rX   rI   rJ   s   @r4   r   r     s    *$&  #ggQ'f- YY$ $ YYP P YY  YY) ) YY		  <C <hsm < 8<$$',+ ||+  eCcN34+  	+ 
 +  +  !%+  
tELL!5tELL7I)I#JJ	K+ ^ ./$#	3S	>*  	$ r5   c                 $   d| j                         v r| d   } |j                         }i }| j                         D ]S  \  }}|j                  d      rd|v r3t	        |j
                  ||   j                  d d d         j
                  }|||<   U |S )Nmodelr   r   r   )r2  r1  itemsendswithr   Tr=   )r1  r^  	target_sdout_dictr   r   s         r4   checkpoint_filter_fnrd  i  s    *//##(
  "IH  " 1::+,"/Yq\5G5G"5MNPPA Or5   c           
      .    | dt         t        dddddd	|S )Nr   zpatch_embed.conv1.convzhead.fc)r   r   )rO      rf  gffffff?)	urlr  meanr+  
first_conv
classifier	pool_size
input_sizecrop_pctr
   )rg  kwargss     r4   _cfgro  x  s1    %#.#  r5   ztimm/iQU  )	hf_hub_idr  )rp  )rO   r  r  )r  r  g      ?)rp  rl  rk  rm  )rO      rq  )   rr  squash)rp  rl  rk  rm  	crop_mode)ztiny_vit_5m_224.dist_in22kz"tiny_vit_5m_224.dist_in22k_ft_in1kztiny_vit_5m_224.in1kztiny_vit_11m_224.dist_in22kz#tiny_vit_11m_224.dist_in22k_ft_in1kztiny_vit_11m_224.in1kztiny_vit_21m_224.dist_in22kz#tiny_vit_21m_224.dist_in22k_ft_in1kztiny_vit_21m_224.in1kz#tiny_vit_21m_384.dist_in22k_ft_in1kz#tiny_vit_21m_512.dist_in22k_ft_in1kc                 r    |j                  dd      }t        t        | |ft        d|      t        d|}|S )Nout_indices)r   r   r7   rO   T)flatten_sequentialrv  )feature_cfgpretrained_filter_fn)popr   r   r  rd  )variant
pretrainedrn  rv  r^  s        r4   _create_tiny_vitr}    sJ    **]L9K  DkJ1 E Lr5   c                 n    t        g dg dg dg dd      }|j                  |       t        d| fi |S )N)@         i@  r  )r7   rN      
   r  r^   r  r  r   r   r"  tiny_vit_5m_224r  updater}  r|  rn  model_kwargss      r4   r  r    s>    &"L -zJ\JJr5   c                 n    t        g dg dg dg dd      }|j                  |       t        d| fi |S )N)r  r     i  r  )r7   rN   r   r   r  r  r  tiny_vit_11m_224r  r  s      r4   r  r    s>    &"L .
KlKKr5   c                 n    t        g dg dg dg dd      }|j                  |       t        d| fi |S )Nr   r   r  i@  r  rO   r  r     r  g?r  tiny_vit_21m_224r  r  s      r4   r  r    s>    & "L .
KlKKr5   c                 n    t        g dg dg dg dd      }|j                  |       t        d| fi |S )Nr  r  r  )r  r  r  r  r  r  tiny_vit_21m_384r  r  s      r4   r  r    >    & %L .
KlKKr5   c                 n    t        g dg dg dg dd      }|j                  |       t        d| fi |S )Nr  r  r  )rr  rr      rr  r  r  tiny_vit_21m_512r  r  s      r4   r  r    r  r5   )rR  rZ  )<r   __all__r   	functoolsr   typingr   r   r   r   r	   r&   torch.nnr!   torch.nn.functional
functionalr   	timm.datar   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   r   	_registryr   r   rt   r   r\  rL   r[   rk   rq   r~   r   r   r   r   rd  ro  default_cfgsr}  r  r  r  r  r  r.  r5   r4   <module>r     sA   +   5 5     AC C * + 1 3 <uxx"" . RYY 2299 $		 2bii <O Od^M299 ^MB  %E9299 E9PCbii CL %"&#
 +/+ ! $($
 ,0, " $($
 ,0, " ,0 Hs,
 ,0 Hsh,[2& 2j
 	K 	K 	L 	L 	L 	L 	L 	L 	L 	Lr5   