
    kh^                        d Z ddlmZ ddlmZmZmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZ G d de	j8                        Z G d de	j8                        Z G d de	j8                        Z G d de	j8                        Z d Z!ed0dee"e"f   de#fd       Z$ G d de	j8                        Z%d0dZ&d1dZ' e e'd       e'dd       e'dd dd!"       e'd       e'dd       e'dd dd!"       e'd       e'dd       e'd       e'd       e'd      d#      Z(ed0d$e%fd%       Z)ed0d$e%fd&       Z*ed0d$e%fd'       Z+ed0d$e%fd(       Z,ed0d$e%fd)       Z-ed0d$e%fd*       Z.ed0d$e%fd+       Z/ed0d$e%fd,       Z0ed0d$e%fd-       Z1ed0d$e%fd.       Z2ed0d$e%fd/       Z3y)2a   CrossViT Model

@inproceedings{
    chen2021crossvit,
    title={{CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification}},
    author={Chun-Fu (Richard) Chen and Quanfu Fan and Rameswar Panda},
    booktitle={International Conference on Computer Vision (ICCV)},
    year={2021}
}

Paper link: https://arxiv.org/abs/2103.14899
Original code: https://github.com/IBM/CrossViT/blob/main/models/crossvit.py

NOTE: model names have been renamed from originals to represent actual input res all *_224 -> *_240 and *_384 -> *_408

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
Modified from Timm. https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
    )partial)ListOptionalTupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tupletrunc_normal__assert   )build_model_with_cfg)register_notrace_function)register_modelgenerate_default_cfgs)BlockCrossVitc                   *     e Zd ZdZd fd	Zd Z xZS )
PatchEmbedz Image to Patch Embedding
    c                 v   t         |           t        |      }t        |      }|d   |d   z  |d   |d   z  z  }|| _        || _        || _        |rG|d   dk(  rt        j                  t        j                  ||dz  ddd      t        j                  d	      t        j                  |dz  |d
z  ddd      t        j                  d	      t        j                  |d
z  |ddd            | _
        y |d   dk(  rt        j                  t        j                  ||dz  ddd      t        j                  d	      t        j                  |dz  |d
z  dd
d      t        j                  d	      t        j                  |d
z  |dd
d            | _
        y y t        j                  ||||      | _
        y )Nr   r               )kernel_sizestridepaddingT)inplace      )r   r   )super__init__r   img_size
patch_sizenum_patchesnn
SequentialConv2dReLUproj)selfr$   r%   in_chans	embed_dim
multi_convr&   	__class__s          P/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/crossvit.pyr#   zPatchEmbed.__init__+   s   X&z*
{jm3zRS}8TU $&!}"MMIIh	QAaYZ[GGD)IIi1ni1n!TU_`aGGD)IIi1niQqZ[\	 A"$MMIIh	QAaYZ[GGD)IIi1ni1n!TU_`aGGD)IIi1niQqZ[\	 % 		(I:V`aDI    c                    |j                   \  }}}}t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       | j                  |      j	                  d      j                  dd      }|S )Nr   zInput image size (*z) doesn't match model (r   z).r    )shaper   r$   r+   flatten	transpose)r,   xBCHWs         r1   forwardzPatchEmbed.forwardG   s    WW
1aT]]1%%$QCq+B4==QRCSBTTUVZVcVcdeVfUggij	lT]]1%%$QCq+B4==QRCSBTTUVZVcVcdeVfUggij	lIIaL  #--a3r2   )   r!   r      F)__name__
__module____qualname____doc__r#   r=   __classcell__r0   s   @r1   r   r   '   s    b8r2   r   c                   .     e Zd Z	 	 	 	 d fd	Zd Z xZS )CrossAttentionc                    t         |           || _        ||z  }|dz  | _        t	        j
                  |||      | _        t	        j
                  |||      | _        t	        j
                  |||      | _        t	        j                  |      | _
        t	        j
                  ||      | _        t	        j                  |      | _        y )Ng      )bias)r"   r#   	num_headsscaler'   LinearwqwkwvDropout	attn_dropr+   	proj_drop)r,   dimrJ   qkv_biasrQ   rR   head_dimr0   s          r1   r#   zCrossAttention.__init__S   s     	")#%
))C84))C84))C84I.IIc3'	I.r2   c                    |j                   \  }}}| j                  |d d dddf         j                  |d| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }||j                  dd      z  | j                  z  }|j                  d      }| j                  |      }||z  j                  dd      j                  |d|      }| j                  |      }| j                  |      }|S )	Nr   r   .r    r   rS   )r5   rM   reshaperJ   permuterN   rO   r7   rK   softmaxrQ   r+   rR   )	r,   r8   r9   Nr:   qkvattns	            r1   r=   zCrossAttention.forwardh   sa   ''1aGGAa1ckN#++Aq$..!t~~BUV^^_`bcefhijGGAJq!T^^Q$..5HIQQRSUVXY[\]GGAJq!T^^Q$..5HIQQRSUVXY[\]AKKB''4::5|||#~~d#AX  A&..q!Q7IIaLNN1r2   )   F        rc   )r@   rA   rB   r#   r=   rD   rE   s   @r1   rG   rG   R   s     /*r2   rG   c                   \     e Zd Zdddddej                  ej
                  f fd	Zd Z xZS )CrossAttentionBlock      @Frc   c
                     t         
|            |	|      | _        t        |||||      | _        |dkD  rt        |      | _        y t        j                         | _        y )N)rJ   rT   rQ   rR   rc   )	r"   r#   norm1rG   ra   r
   r'   Identity	drop_path)r,   rS   rJ   	mlp_ratiorT   rR   rQ   rj   	act_layer
norm_layerr0   s             r1   r#   zCrossAttentionBlock.__init__}   sV     	_
"
	 1:B),BKKMr2   c                 ~    |d d dddf   | j                  | j                  | j                  |                  z   }|S )Nr   r   .)rj   ra   rh   )r,   r8   s     r1   r=   zCrossAttentionBlock.forward   s9    a1ckNT^^DIIdjjm,DEEr2   )	r@   rA   rB   r'   GELU	LayerNormr#   r=   rD   rE   s   @r1   re   re   {   s-     gg||R0r2   re   c                        e Zd Zddddej                  ej
                  f fd	Zdeej                     deej                     fdZ
 xZS )MultiScaleBlockFrc   c                    t         |           t        |      }|| _        t	        j
                         | _        t        |      D ]  }g }t        ||         D ]0  }|j                  t        ||   ||   ||   ||||	|   |             2 t        |      dk7  sU| j                  j                  t	        j                  |         t        | j                        dk(  rd | _        t	        j
                         | _        t        |      D ]r  }||   ||dz   |z     k(  r	  |||          |
       t	        j                  ||   ||dz   |z           g}| j                  j                  t	        j                  |        t t	        j
                         | _        t        |      D ]  }|dz   |z  }||   }|d   dk(  r6| j                  j                  t        ||   |||   ||||	d   |             Ng }t        |d         D ]-  }|j                  t        ||   |||   ||||	d   |             / | j                  j                  t	        j                  |         t	        j
                         | _        t        |      D ]x  }||dz   |z     ||   k(  r	  |||dz   |z            |
       t	        j                  ||dz   |z     ||         g}| j                   j                  t	        j                  |        z y )N)rS   rJ   rk   rT   rR   rQ   rj   rm   r   r   rX   )r"   r#   lennum_branchesr'   
ModuleListblocksrangeappendr   r(   projsri   rL   fusionre   revert_projs)r,   rS   patchesdepthrJ   rk   rT   rR   rQ   rj   rl   rm   ru   dtmpid_nh_r0   s                      r1   r#   zMultiScaleBlock.__init__   s    	3x(mmo|$ 	8AC58_ 


5A'l'l%'''l)	 	
 3x1}""2==##67	8  t{{q DK]]_
|$ 	3A1va!e|344 "#a&)9;		#a&#qSTuXdNdJe8fgJJbmmS12	3 mmo|$ 	8Aa%<'B2BRyA~""'G"$"+A,!)"+"+"+B-#-	
 uRy) 
AJJ2G"$"+A,!)"+"+"+B-#-	  	
 ""2==##679	8< MMO|$ 	:AAE\)*c!f4 "#q1u&<"=>	yya!e|%;!<c!fEG$$R]]C%89	:r2   r8   returnc           	         g }t        | j                        D ]  \  }}|j                   |||                ! t        j                  j                  t        t        j                     g       }t        | j                        D ])  \  }}|j                   |||   d d dddf                + g }t        t        | j                  | j                              D ]  \  }\  }}	t        j                  ||   ||dz   | j                  z     d d dd df   fd      }
 ||
      }
 |	|
d d dddf         }t        j                  |||   d d dd df   fd      }
|j                  |
        |S )Nr   r   .rY   )	enumeraterw   ry   torchjitannotater   Tensorrz   zipr{   r|   catru   )r,   r8   outs_br   blockproj_cls_tokenr+   outsr{   revert_projr   reverted_proj_cls_tokens               r1   r=   zMultiScaleBlock.forward   sq   !$++. 	'HAuMM%!+&	' ++D,>C , 	@GAt!!$vayAaC'=">?	@ (1#dkk4CTCT2U(V 	$A$))^A.AARAR7R0STUWXWY[^T^0_`fghC+C&1#a1ck2B&C#))4fQi12s
6KLRSTCKK	 r2   )r@   rA   rB   r'   ro   rp   r#   r   r   r   r=   rD   rE   s   @r1   rr   rr      sK     gg||U:nell+ U\\0B r2   rr   c                 j    t        | |      D cg c]  \  }}|d   |z  |d   z  |z   c}}S c c}}w )Nr   r   )r   )r$   r}   r   ps       r1   _compute_num_patchesr   	  s7    .1(G.DEdaAaDAI!!EEEs   /ss
crop_scalec                 z   | j                   dd \  }}||d   k7  s||d   k7  r|ri|d   |k  ra|d   |k  rYt        t        ||d   z
  dz              t        t        ||d   z
  dz              }}| dddd|||d   z   |||d   z   f   } | S t        j                  j
                  j                  | |dd      } | S )	a~  
    Pulled out of CrossViT.forward_features to bury conditional logic in a leaf node for FX tracing.
    Args:
        x (Tensor): input image
        ss (tuple[int, int]): height and width to scale to
        crop_scale (bool): whether to crop instead of interpolate to achieve the desired scale. Defaults to False
    Returns:
        Tensor: the "scaled" image batch tensor
    rW   Nr   r          @bicubicF)sizemodealign_corners)r5   introundr   r'   
functionalinterpolate)r8   r   r   r;   r<   cucls          r1   scale_imager     s     7723<DAqBqEzQ"Q%Z"Q%1*A!BqE	R/013ua"Q%i2=M7N3OB!Q21:r"r!u*}45A H ##//Z_/`AHr2   c                       e Zd ZdZdddddddd	d
dddddddd eej                  d      df fd	Zd Ze	j                  j                  d        Ze	j                  j                  d d       Ze	j                  j                  d!d       Ze	j                  j                  dej                  fd       Zd"dedee   fdZdee	j,                     fdZd dee	j,                     dede	j,                  fdZd Z xZS )#r   zI Vision Transformer with support for patch or hybrid CNN input stage
    r>   )      ?r   )rb   r!   r     )     )r   r   r   r   r   )   r   )r   r   rf   FTrc   gư>)epstokenc                    t         $|           |dv sJ || _        || _        t	        |      | _        t	        |      }|D cg c]1  }t        | j
                  D cg c]  }t        ||z         c}      3 c}}| _        || _	        t        | j                  |      }t        |      | _        || _        t        |      x| _        | _        t#        j$                         | _        t)        | j                        D ]~  }t+        | d| t#        j,                  t/        j0                  dd||   z   ||                      t+        | d| t#        j,                  t/        j0                  dd||                       t3        | j                  ||      D ]/  \  }}}| j&                  j5                  t7        |||||
             1 t#        j8                  |      | _        t        |D cg c]  }t        |dd         c}      }t/        j<                  d||      D cg c]  }|j?                          }}d}t#        j$                         | _         tC        |      D ]V  \  }} tE        | d d	       | d	   z   }!||||!z    }"tG        ||| ||	||||"|

      }#||!z  }| j@                  j5                  |#       X t#        j$                  t)        | j                        D cg c]  } |||          c}      | _$        t#        j8                  |      | _%        t#        j$                  t)        | j                        D cg c]4  }|dkD  rt#        jL                  ||   |      nt#        jN                         6 c}      | _(        t)        | j                        D ]6  }tS        tU        | d|       d       tS        tU        | d|       d       8 | jW                  | jX                         y c c}w c c}}w c c}w c c}w c c}w c c}w )Nr   avg
pos_embed_r   
cls_token_)r$   r%   r-   r.   r/   )r   rW   r   rX   )rJ   rk   rT   rR   rQ   rj   rm   {Gz?std)-r"   r#   num_classesglobal_poolr   r$   tupler   img_size_scaledr   r   rt   ru   r.   sumnum_featureshead_hidden_sizer'   rv   patch_embedrx   setattr	Parameterr   zerosr   ry   r   rP   pos_droplinspaceitemrw   r   maxrr   norm	head_droprL   ri   headr   getattrapply_init_weights)%r,   r$   	img_scaler%   r-   r   r.   r~   rJ   rk   r/   r   rT   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterm   r   sisjr&   r   im_sr   r   r8   total_depthdprdpr_ptridx	block_cfg
curr_depthdpr_blkr0   s%                                       r1   r#   zCrossVit.__init__&  s   , 	....&&!(+i(	XabRTdmm&Ls27|&L Mb$*4+?+?L
O"47	NBD1==? t(() 	[ADJqc*BLLQKXYNHZ\efg\h9i,jkDJqc*BLLQ9UV<9X,YZ	[ d22J	J 	JD!Q##! %)	 

]3613qv;67!&>;!OPAqvvxPPmmo'. 	$NCYs^,y}<Jww34D!##!((%C z!GKKs#!	$$ MMU4K\K\E]"^:il#;"^_	I.MM4,,-#/ 5@!OBIIilK0V#/ 0	 t(() 	DA'$*QC(89sC'$*QC(89sC	D 	

4%%&s 'Mb2 7P* #_#/s0   O/O*1	O/*O5O:O?;9P*O/c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r   r   )	
isinstancer'   rL   r   weightrI   init	constant_rp   )r,   ms     r1   r   zCrossVit._init_weights~  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r2   c                     t               }t        | j                        D ]J  }|j                  d|        t	        | d| d       }|*|j
                  s7|j                  d|        L |S )Nr   r   )setrx   ru   addr   requires_grad)r,   outr   pes       r1   no_weight_decayzCrossVit.no_weight_decay  sr    et(() 	*AGGj$%A3/6B~""2"2*QC()		*
 
r2   c                      t        dddg      S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemrw   )dict)r,   coarses     r1   group_matcherzCrossVit.group_matcher  s    4-/CD
 	
r2   c                     |rJ d       y )Nz$gradient checkpointing not supported )r,   enables     r1   set_grad_checkpointingzCrossVit.set_grad_checkpointing  s    AAAz6r2   r   c                     | j                   S N)r   )r,   s    r1   get_classifierzCrossVit.get_classifier  s    yyr2   r   r   c           	      $   || _         ||dv sJ || _        t        j                  t	        | j
                        D cg c]>  }|dkD  r#t        j                  | j                  |   |      nt        j                         @ c}      | _	        y c c}w )Nr   r   )
r   r   r'   rv   rx   ru   rL   r.   ri   r   )r,   r   r   r   s       r1   reset_classifierzCrossVit.reset_classifier  s    &""2222*DMM4,,-#
 :EqBIIdnnQ'5bkkm[#
 	 #
s   ABc                    |j                   d   }g }t        | j                        D ]  \  }}|}| j                  |   }t	        ||| j
                        } ||      }|dk(  r| j                  n| j                  }|j                  |dd      }t        j                  ||fd      }|dk(  r| j                  n| j                  }	||	z   }| j                  |      }|j                  |        t        | j                        D ]  \  }}
 |
|      } t        | j                         D cg c]  \  }} |||          }}}|S c c}}w )Nr   rX   r   rY   )r5   r   r   r   r   r   cls_token_0cls_token_1expandr   r   pos_embed_0pos_embed_1r   ry   rw   r   )r,   r8   r9   xsr   r   x_r   
cls_tokens	pos_embedr   r   s               r1   forward_featureszCrossVit.forward_features  s:   GGAJ'(8(89 	NA{B%%a(BRT__5BRB-.!V))9I9IJ#**1b"5JJ+3B,-F((8H8HIiBr"BIIbM	  , 	FAsRB	 *3499)=>gad2a5k>>	 ?s   )Er   
pre_logitsc                 b   | j                   dk(  r)|D cg c]  }|d d dd f   j                  d       c}n|D cg c]  }|d d df    c}}|D cg c]  }| j                  |       }}|s't        | j                  d   t
        j                        r%t        j                  |D cg c]  }| c}d      S t        j                  t        j                  t        | j                        D cg c]  \  }} |||          c}}d      d      S c c}w c c}w c c}w c c}w c c}}w )Nr   r   rY   r   )r   meanr   r   r   r'   ri   r   r   stackr   )r,   r   r  r8   r   r   s         r1   forward_headzCrossVit.forward_head  s    373C3Cu3LB/qa12hmmm"/dfRg_`STUVXYUYSZRg)+,AdnnQ,,DIIaL"++>99_Aa_!44zz%++)DIIBV&Wwq$tBqE{&W]^_efgg	 0Rg,,&Ws   "DDD!*	D&2D+c                 J    | j                  |      }| j                  |      }|S r   )r  r  )r,   r8   r   s      r1   r=   zCrossVit.forward  s'    ""1%b!r2   F)Tr   )r@   rA   rB   rC   r   r'   rp   r#   r   r   r   ignorer   r   r   Moduler   r   r   strr   r   r   r  boolr  r=   rD   rE   s   @r1   r   r   "  s>   
   3"r||6)V'p- YY  YY
 
 YYB B YY		  C hsm T%,,%7 .htELL1 ht hPUP\P\ hr2   c                 j    |j                  dd       rt        d      d }t        t        | |fd|i|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.c                 ~    i }| j                         D ]'  }d|v sd|v r|j                  dd      }n|}| |   ||<   ) |S )Nr  	cls_token.r   )keysreplace)
state_dictnew_state_dictkeynew_keys       r1   pretrained_filter_fnz._create_crossvit.<locals>.pretrained_filter_fn  sW    ??$ 	6Cc![C%7++c3/&0oN7#	6 r2   r  )getRuntimeErrorr   r   )variant
pretrainedkwargsr  s       r1   _create_crossvitr    sM    zz/4(YZZ   2	
  r2   c                 0    | ddd dt         t        dddd
|S )Nr   )r      r!  g      ?T)zpatch_embed.0.projzpatch_embed.1.proj)zhead.0zhead.1)
urlr   
input_size	pool_sizecrop_pctr  r   fixed_input_size
first_conv
classifierr   )r"  r  s     r1   _cfgr)    s2    =tY^%.BX\B*  r2   ztimm/)	hf_hub_id)zpatch_embed.0.proj.0zpatch_embed.1.proj.0)r*  r'  )r     r+  r   )r*  r#  r'  r%  )zcrossvit_15_240.in1kzcrossvit_15_dagger_240.in1kzcrossvit_15_dagger_408.in1kzcrossvit_18_240.in1kzcrossvit_18_dagger_240.in1kzcrossvit_18_dagger_408.in1kzcrossvit_9_240.in1kzcrossvit_9_dagger_240.in1kzcrossvit_base_240.in1kzcrossvit_small_240.in1kzcrossvit_tiny_240.in1kr   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr   g?r   r!   `   r   r   r   r   r   r   r   r   r   r%   r.   r~   rJ   rk   crossvit_tiny_240r  r  r   r   r  r  r  
model_argsmodels       r1   r2  r2    sY     b"X"cS\^girRsa&I/J n%8ZnSWXbSmflSmnELr2   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr-  r   r!   r   r   r/  r   r0  r1  crossvit_small_240r3  r   r4  r5  s       r1   r9  r9    sY     b"X#sT]_hjsSta&I/J o%9joTXYcTngmTnoELr2   c           	      z    t        dddgddgg dg dg dgddgg d      }t        dd	| d
t        |fi |}|S )Nr-  r   r!   r   r?   r/  r0  r1  crossvit_base_240r3  r   r4  r5  s       r1   r;  r;     sY     b"X#sT]_hjsStr(i1J n%8ZnSWXbSmflSmnELr2   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr-  r   r!         r   r   r   r   r   r   r   r1  crossvit_9_240r3  r   r4  r5  s       r1   rA  rA  )  sY     b"X#sT]_hjsSta&I/J k%5*kPTU_PjciPjkELr2   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr-  r   r!   r   r   r      r   r   r@  r1  crossvit_15_240r3  r   r4  r5  s       r1   rE  rE  2  sY     b"X#sT]_hjsSta&I/J l%6:lQUV`QkdjQklELr2   c           
      |    t        ddddgddgg dg dg dgddgg dd	|}t        dd
| dt        |fi |}|S )Nr-  r   r!   r>     r   r   r   r   r@  r1  crossvit_18_240r3  r   r4  r5  s       r1   rI  rI  ;  sf     9"BxC:V_ajluUva&I9179J l%6:lQUV`QkdjQklELr2   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr-  r   r!   r=  r>  r?  r   r@  Tr   r%   r.   r~   rJ   rk   r/   crossvit_9_dagger_240r3  r   r4  r5  s       r1   rL  rL  D  s\    "BxC:V_ajluUva&I$@J r%<rW[\fWqjpWqrELr2   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr-  r   r!   r   r   rC  r   r@  TrK  crossvit_15_dagger_240r3  r   r4  r5  s       r1   rN  rN  M  \     b"X#sT]_hjsSta&I$@J s%=*sX\]gXrkqXrsELr2   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr   g?r   r!   r   r   rC  r   r@  TrK  crossvit_15_dagger_408r3  r   r4  r5  s       r1   rR  rR  V  rO  r2   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr-  r   r!   r>   rG  rH  r   r@  TrK  crossvit_18_dagger_240r3  r   r4  r5  s       r1   rT  rT  _  rO  r2   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )NrQ  r   r!   r>   rG  rH  r   r@  TrK  crossvit_18_dagger_408r3  r   r4  r5  s       r1   rV  rV  h  rO  r2   r	  ) )4rC   	functoolsr   typingr   r   r   r   torch.nnr'   	timm.datar   r	   timm.layersr
   r   r   r   _builderr   _features_fxr   	_registryr   r   vision_transformerr   __all__r  r   rG   re   rr   r   r   r  r   r   r  r)  default_cfgsr2  r9  r;  rA  rE  rI  rL  rN  rR  rT  rV  r   r2   r1   <module>rc     s  ,  ( (   A C C * 3 < %,( (V&RYY &R")) >lbii l^F uS#X D  (iryy iX. % 73#'C$ $( -]hk$ !73#'C$ $( -]hk$  '2"&C# #W5#g6"W55& < X   h   X   (   8   8      (   (   (   (  r2   