
    khk                        d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlm Z  ddl!m"Z"m#Z# dgZ$dddddZ%dddddZ&dddddZ' G d dejP                        Z) G d de
j                  jP                        Z* G d d e
j                  jP                        Z+ G d! d"e
j                  jP                        Z, G d# d$ejP                        Z- G d% d&ejP                        Z. G d' d(ejP                        Z/ G d) d*ejP                        Z0 G d+ d,ejb                        Z2 G d- d.ejP                        Z3 G d/ dejP                        Z4d:d0Z5 e" e5d12       e5d12       e5d12       e5d12      d3      Z6d;d4Z7e#d;d5e4fd6       Z8e#d;d5e4fd7       Z9e#d;d5e4fd8       Z:e#d;d5e4fd9       Z;y)<aJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
    N)partial)DictListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)create_conv2dcreate_norm_layerget_act_layerget_norm_layerConvNormAct)DropPathtrunc_normal_	to_2tuple	to_ntuplendgrid   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelEfficientFormerV2)(   P        )    @      i   )r!   0   x      )r!   r$   `      )LS2S1S0)   r-      
   )   r0         )   r3   	      )   r6   r5   r0   )r0   r0   )r0   r0   r0   r0   r3   r3   r3   r3   r3   r3   r3   r0   r0   r0   r0   )
r0   r0   r0   r3   r3   r3   r3   r0   r0   r0   )r0   r0   )r0   r0   r3   r3   r3   r3   r3   r3   r0   r0   r0   r0   )r0   r0   r3   r3   r3   r3   r0   r0   )r0   r0   )	r0   r0   r3   r3   r3   r3   r0   r0   r0   )r0   r0   r3   r3   r0   r0   )r0   r0   )r0   r3   r3   r3   r0   r0   )r0   r3   r3   r0   c                   6     e Zd Z	 	 	 	 	 	 	 	 d fd	Zd Z xZS )ConvNormc           
          |
xs i }
t         t        |           t        ||||||||      | _        t        |	|fi |
| _        y )N)stridepaddingdilationgroupsbias)superr8   __init__r   convr   bn)selfin_channelsout_channelskernel_sizer:   r;   r<   r=   r>   
norm_layernorm_kwargs	__class__s              Z/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/efficientformer_v2.pyr@   zConvNorm.__init__:   sU     "'Rh&(!	
	 $JLL    c                 J    | j                  |      }| j                  |      }|S N)rA   rB   rC   xs     rJ   forwardzConvNorm.forwardU   s!    IIaLGGAJrK   )r   r    r   r   Tbatchnorm2dN__name__
__module____qualname__r@   rP   __classcell__rI   s   @rJ   r8   r8   9   s(    
 $M6rK   r8   c                        e Zd ZU eeej                  f   ed<   dddddej                  df fd	Z
 ej                         d fd		       Zd
ej                  dej                  fdZd Z xZS )Attention2dattention_bias_cacher    r!   r2   r0      Nc           	      f   t         |           || _        |dz  | _        || _        t        |      }|at        |D cg c]  }t        j                  ||z         c}      }t        ||d||      | _
        t        j                  |d      | _        nd | _
        d | _        || _        | j                  d   | j                  d   z  | _        t!        ||z        | _        t!        ||z        |z  | _        || _        | j                  | j                  z  }	t        ||	      | _        t        ||	      | _        t        || j$                        | _        t        | j$                  | j$                  d| j$                        | _        t        j0                  | j                  | j                  d	      | _        t        j0                  | j                  | j                  d	      | _         |       | _        t        | j$                  |d      | _        t;        j<                  t?        t;        j@                  | j                  d         t;        j@                  | j                  d                     jC                  d      }
|
d
d d d f   |
d
d d d f   z
  jE                         }|d   | j                  d   z  |d   z   }t:        j                  jG                  t;        jH                  || j                              | _%        | jM                  dt;        jN                  |      d       i | _(        y c c}w )N      r3   rF   r:   r=   bilinear)scale_factormoder   r   )rF   r=   )rF   .attention_bias_idxsF
persistent))r?   r@   	num_headsscalekey_dimr   tuplemathceilr8   stride_convnnUpsampleupsample
resolutionNintddh
attn_ratioqkvv_localConv2dtalking_head1talking_head2actprojtorchstackr   arangeflattenabs	Parameterzerosattention_biasesregister_buffer
LongTensorr[   )rC   dimrh   rf   ru   rp   	act_layerr:   rkhposrel_posrI   s               rJ   r@   zAttention2d.__init__^   s    	"_
z*
zJ!		!f* 5JKJ'SaWZ[DKKV*MDM#D DM$#dooa&88Z')*j7*+i7$\\DNN*#r"#r"#tww'aPYYt~~t~~STUYYt~~t~~STU;TWWc1-	kk&dooa.@!A5<<PTP_P_`aPbCcdemmnopsAt|$s3a<'88==?1: 22gaj@ % 2 25;;y$&&3Q R2E4D4DW4MZ_`$&!;  Ks   L.c                 R    t         |   |       |r| j                  ri | _        y y y rM   r?   trainr[   rC   rb   rI   s     rJ   r   zAttention2d.train   )    dD--(*D% .4rK   devicereturnc                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rM   r   jit
is_tracingtrainingr   rc   strr[   rC   r   
device_keys      rJ   get_attention_biasesz Attention2d.get_attention_biases       99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88rK   c                 $   |j                   \  }}}}| j                  | j                  |      }| j                  |      j                  || j                  d| j
                        j                  dddd      }| j                  |      j                  || j                  d| j
                        j                  dddd      }| j                  |      }| j                  |      }	|j                  || j                  d| j
                        j                  dddd      }||z  | j                  z  }
|
| j                  |j                        z   }
| j                  |
      }
|
j                  d      }
| j                  |
      }
|
|z  j!                  dd      }|j                  || j"                  | j$                  d   | j$                  d         |	z   }| j&                  | j'                  |      }| j)                  |      }| j+                  |      }|S Nr   r   r3   r6   r   )shaperl   rv   reshaperf   rq   permuterw   rx   ry   rg   r   r   r{   softmaxr|   	transposert   rp   ro   r}   r~   rC   rO   BCHWrv   rw   rx   ry   attns              rJ   rP   zAttention2d.forward   s   WW
1a'  #AFF1IaTVV<DDQ1aPFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aHA#d//99!!$'|||#!!$'AX  A&IIa$//!"4dooa6HIGS==$a AHHQKIIaLrK   TrT   rU   rV   r   r   r   Tensor__annotations__rm   GELUr@   no_gradr   r   r   rP   rW   rX   s   @rJ   rZ   rZ   [   st    sELL011 gg.'` U]]_+ +
95<< 9ELL 9rK   rZ   c                   $     e Zd Z fdZd Z xZS )LocalGlobalQueryc                     t         |           t        j                  ddd      | _        t        j
                  ||ddd|      | _        t        ||d      | _        y )Nr   r6   r   r3   )rF   r:   r;   r=   )	r?   r@   rm   	AvgPool2dpoolrz   localr8   r~   )rC   in_dimout_dimrI   s      rJ   r@   zLocalGlobalQuery.__init__   sN    LLAq)	YYvv1QPQZ`a
VWa0	rK   c                 v    | j                  |      }| j                  |      }||z   }| j                  |      }|S rM   )r   r   r~   )rC   rO   local_qpool_qrv   s        rJ   rP   zLocalGlobalQuery.forward   s8    **Q-1fIIaLrK   rS   rX   s   @rJ   r   r      s    1rK   r   c                        e Zd ZU eeej                  f   ed<   ddddddej                  f fd	Z
 ej                         d fd		       Zd
ej                  dej                  fdZd Z xZS )Attention2dDownsampler[   r       r2   r0   r\   Nc           
      ^   t         |           || _        |dz  | _        || _        t        |      | _        t        | j                  D cg c]  }t        j                  |dz         c}      | _
        | j                  d   | j                  d   z  | _        | j                  d   | j                  d   z  | _        t        ||z        | _        t        ||z        |z  | _        || _        |xs || _        | j                  | j                  z  }	t%        ||	      | _        t)        ||	d      | _        t)        || j                  d      | _        t)        | j                  | j                  dd| j                        | _         |       | _        t)        | j                  | j"                  d      | _        t5        j6                  t9        j:                  || j                              | _        t9        j>                  tA        t9        jB                  | j                  d         t9        jB                  | j                  d                     jE                  d      }
t9        j>                  tA        t9        jB                  d| j                  d   d      t9        jB                  d| j                  d   d                  jE                  d      }|dd d d f   |
dd d d f   z
  jG                         }|d   | j                  d   z  |d   z   }| jI                  d	|d
       i | _%        y c c}w )Nr^   r6   r   r   r3   r_   )step.rc   Frd   )&r?   r@   rf   rg   rh   r   rp   ri   rj   rk   resolution2rq   N2rr   rs   rt   ru   r   r   rv   r8   rw   rx   ry   r}   r~   rm   r   r   r   r   r   r   r   r   r   r   r[   )rC   r   rh   rf   ru   rp   r   r   r   r   k_posq_posr   rI   s                rJ   r@   zAttention2dDownsample.__init__   s    	"_
#J/ DOO!Lq$))AE"2!LM#dooa&88""1%(8(8(;;Z')*j7*+i7$~#\\DNN*!#r*#r1%#tww*aRVRYRYZ;TWWdllA6	 "U[[DFF-K LF5<<0B#CU\\RVRaRabcRdEefgoopqrFLLDOOA.Q7LLDOOA.Q7
  71: 	 a&sD!|)<<AAC1: 22gaj@2GN$&!7 "Ms   L*c                 R    t         |   |       |r| j                  ri | _        y y y rM   r   r   s     rJ   r   zAttention2dDownsample.train   r   rK   r   r   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rM   r   r   s      rJ   r   z*Attention2dDownsample.get_attention_biases   r   rK   c                 l   |j                   \  }}}}| j                  |      j                  || j                  d| j                        j                  dddd      }| j                  |      j                  || j                  d| j                        j                  dddd      }| j                  |      }| j                  |      }	|j                  || j                  d| j                        j                  dddd      }||z  | j                  z  }
|
| j                  |j                        z   }
|
j                  d      }
|
|z  j                  dd      }|j                  || j                  | j                   d   | j                   d         |	z   }| j#                  |      }| j%                  |      }|S r   )r   rv   r   rf   r   r   rw   rq   rx   ry   rg   r   r   r   r   rt   r   r}   r~   r   s              rJ   rP   zAttention2dDownsample.forward  sk   WW
1aFF1IaTWW=EEaAqQFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aHA#d//99|||#AX  A&IIa$"2"21"5t7G7G7JKgUHHQKIIaLrK   r   r   rX   s   @rJ   r   r      st    sELL011 gg+'Z U]]_+ +
95<< 9ELL 9rK   r   c                   \     e Zd Zdddddej                  ej
                  f fd	Zd Z xZS )
Downsampler3   r6   r   r\   Fc
                     t         
|           t        |      }t        |      }t        |      }|	xs t        j                         }	t        ||||||	      | _        |rt        ||||      | _        y d | _        y )N)rF   r:   r;   rG   )r   r   rp   r   )	r?   r@   r   rm   Identityr8   rA   r   r   )rC   in_chsout_chsrF   r:   r;   rp   use_attnr   rG   rI   s             rJ   r@   zDownsample.__init__  s     	,6"G$02;;=
#!
	 -%#	DI DIrK   c                 h    | j                  |      }| j                  | j                  |      |z   S |S rM   )rA   r   )rC   rO   outs      rJ   rP   zDownsample.forward@  s1    iil99 99Q<#%%
rK   	rT   rU   rV   rm   r   BatchNorm2dr@   rP   rW   rX   s   @rJ   r   r     s-    
 gg~~#JrK   r   c                   ^     e Zd ZdZddej
                  ej                  ddf fd	Zd Z xZ	S )ConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    N        Fc           	      \   t         |           |xs |}|xs |}t        ||dd||      | _        |rt        ||d|d||      | _        nt        j                         | _        t        j                  |      | _        t        ||d|      | _
        t        j                  |      | _        y )Nr   T)r>   rG   r   r3   )r=   r>   rG   r   )rG   )r?   r@   r   fc1midrm   r   Dropoutdrop1r8   fc2drop2)	rC   in_featureshidden_featuresout_featuresr   rG   dropmid_convrI   s	           rJ   r@   zConvMlpWithNorm.__init__M  s     	#2{)8[!*	C "!&TjT]_DH {{}DHZZ%
O\1TZZ%
rK   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S rM   )r   r   r   r   r   rN   s     rJ   rP   zConvMlpWithNorm.forwardg  sH    HHQKHHQKJJqMHHQKJJqMrK   )
rT   rU   rV   __doc__rm   r   r   r@   rP   rW   rX   s   @rJ   r   r   G  s.     !gg~~&4rK   r   c                   &     e Zd Zd fd	Zd Z xZS )LayerScale2dc                     t         |           || _        t        j                  |t        j                  |      z        | _        y rM   )r?   r@   inplacerm   r   r   onesgamma)rC   r   init_valuesr   rI   s       rJ   r@   zLayerScale2d.__init__q  s2    \\+

3"?@
rK   c                     | j                   j                  dddd      }| j                  r|j                  |      S ||z  S )Nr   r   )r   viewr   mul_)rC   rO   r   s      rJ   rP   zLayerScale2d.forwardv  s7    

2q!, $qvve};!e);rK   )h㈵>FrS   rX   s   @rJ   r   r   p  s    A
<rK   r   c            	       `     e Zd Zdej                  ej
                  ddddddf	 fd	Zd Z xZS )	EfficientFormerV2Block      @r   r   r\   NTc                 
   t         |           |
ret        ||||	      | _        |t	        ||      nt        j                         | _        |dkD  rt        |      nt        j                         | _	        nd | _        d | _        d | _	        t        |t        ||z        |||d      | _        |t	        ||      nt        j                         | _        |dkD  rt        |      | _        y t        j                         | _        y )N)rp   r   r:   r   T)r   r   r   rG   r   r   )r?   r@   rZ   token_mixerr   rm   r   ls1r   
drop_path1r   rr   mlpls2
drop_path2)rC   r   	mlp_ratior   rG   	proj_drop	drop_pathlayer_scale_init_valuerp   r:   r   rI   s              rJ   r@   zEfficientFormerV2Block.__init__|  s     	*%#	 D 1G0R $+-XZXcXcXe H5>^hy1DO#DDH"DO"i0!
 -C,N  ')TVT_T_Ta 	1:R(9-R[[]rK   c                     | j                   2|| j                  | j                  | j                  |                  z   }|| j                  | j	                  | j                  |                  z   }|S rM   )r   r   r   r   r   r   rN   s     rJ   rP   zEfficientFormerV2Block.forward  s^    'DOODHHT-=-=a-@$ABBA! 566rK   r   rX   s   @rJ   r   r   {  s4     gg~~#'(STrK   r   c                   L     e Zd Zej                  ej
                  f fd	Z xZS )Stem4c           
          t         |           d| _        t        ||dz  dddd||      | _        t        |dz  |dddd||      | _        y )Nr0   r6   r3   r   T)rF   r:   r;   r>   rG   r   )r?   r@   r:   r   conv1conv2)rC   r   r   r   rG   rI   s        rJ   r@   zStem4.__init__  s[     GqLa14!Y

 !qL'qAD!Y

rK   )rT   rU   rV   rm   r   r   r@   rW   rX   s   @rJ   r  r    s    24''bnn 

 

rK   r  c                   f     e Zd Zddddddddddej                  ej
                  f fd		Zd
 Z xZS )EfficientFormerV2Stager\   TNFr   r   r   r   c                    t         |           d| _         t        |      |
      }
t	        |      }|rHt        ||||||      | _        |}t        |D cg c]  }t        j                  |dz         c}      }n ||k(  sJ t        j                         | _        g }t        |      D ]2  }||	z
  dz
  }t        ||||
|   |xr ||kD  |||   |||
      }||gz  }4 t        j                  | | _        y c c}w )NF)r   rp   rG   r   r6   r   )	rp   r:   r   r   r   r   r   r   rG   )r?   r@   grad_checkpointingr   r   r   
downsampleri   rj   rk   rm   r   ranger   
Sequentialblocks)rC   r   dim_outdepthrp   r  block_stridedownsample_use_attnblock_use_attnnum_vitr   r   r   r   r   rG   r   r  	block_idx
remain_idxbrI   s                        rJ   r@   zEfficientFormerV2Stage.__init__  s   & 	"'$Ie$Y/	z*
(,%%#DO C*EQ		!a% 0EFJ'>!> kkmDOu 	I1,J&%##I.'BI
,B##I.'=#%A qcMF	 mmV,+  Fs   C7c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S rM   )r  r
  r   r   is_scriptingr   r  rN   s     rJ   rP   zEfficientFormerV2Stage.forward  sS    OOA""599+A+A+Ct{{A.A  AArK   r   rX   s   @rJ   r  r    s<      % #'gg~~!7-rrK   r  c                       e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zd Zej                  j                  d        Zej                  j                  dd       Z	ej                  j                  dd       Z
ej                  j                  dej                  fd       Zdded	ee   fd
Zej                  j                  dd       Z	 	 	 	 	 ddej&                  deeeee   f      dededededeeej&                     eej&                  eej&                     f   f   fdZ	 	 	 ddeeee   f   dedefdZd ZddefdZd Z xZS ) r   c                 n   t         |           |dv sJ || _        || _        g | _        t        |      }t        t        |      |	      }t        |
      }
t        ||d   |
|      | _
        |d   }d}t        |      }t        j                  d|t        |            j                  |      D cg c]  }|j!                          }}|xs ddt        |      dz
  z  z   } t#        |      |      }g }t%        |      D ]  }t'        |D cg c]  }t)        j*                  ||z         c}      }t-        |||   ||   |||   |d	k(  rd	nd |d
k\  |d	k\  |||   |||   ||
|      }||   r|d	z  }||   }| xj                  t/        ||d|       gz  c_        |j1                  |        t3        j4                  | | _        |d   x| _        | _         ||d         | _        t3        j>                  |      | _         |dkD  rt3        jB                  |d   |      nt3        jD                         | _#        || _$        | jH                  r8|dkD  rt3        jB                  |d   |      nt3        jD                         | _%        nd | _%        | jM                  | jN                         d| _(        y c c}w c c}w )N)avgrQ   )epsr   )r   rG   r0   Fr   r   r6   r3   )r  rp   r  r  r  r  r  r   r   r   r   r   rG   zstages.)num_chs	reductionmoduler   F))r?   r@   num_classesglobal_poolfeature_infor   r   r   r   r  stemlenr   linspacesumsplittolistr   r  ri   rj   rk   r  dictappendrm   r  stagesnum_featureshead_hidden_sizenormr   	head_dropLinearr   headdist	head_distapplyinit_weightsdistilled_training)rC   depthsin_chansimg_sizer#  
embed_dimsdownsamples
mlp_ratiosrG   norm_epsr   r"  	drop_rateproj_drop_ratedrop_path_rater   r  distillationprev_dimr:   
num_stagesrO   dprr-  iscurr_resolutionstagerI   s                               rJ   r@   zEfficientFormerV2.__init__   s   ( 	k)))&&X&^J7XF
!),	(JqMYS]^	a=[
#(>>!^S[#Q#W#WX^#_`aqxxz``!KX3v;?0K%K*Yz*:6
z" 	!A#H$MqTYYq6z%:$MNO*1Qi*&q>"#q&Qd$%F Av$Q-(a&'=#%E" 1~!!!}H$x6T[\][^R_"`!aaMM% /	!0 mmV, 5?rNBD1z"~.	I.>IAoBIIjnk:SUS^S^S`	 	99GRUVRYYz"~{C\^\g\g\iDN!DN

4$$%"'U a
 %Ns   1J-J2
c                     t        |t        j                        rOt        |j                  d       |j
                  +t        j                  j                  |j
                  d       y y y )N{Gz?)stdr   )
isinstancerm   r2  r   weightr>   init	constant_)rC   ms     rJ   r7  zEfficientFormerV2.init_weightsO  sJ    a#!((,vv!!!!&&!, " $rK   c                 ^    | j                         D ch c]  \  }}d|v s| c}}S c c}}w )Nr   )named_parameters)rC   rw   _s      rJ   no_weight_decayz!EfficientFormerV2.no_weight_decayU  s+    "335Qda9Kq9PQQQs   ))c                 $    t        dddg      }|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )r%  r  )r+  )rC   coarsematchers      rJ   group_matcherzEfficientFormerV2.group_matcherY  s    -/CD
 rK   c                 4    | j                   D ]	  }||_         y rM   )r-  r
  )rC   enablerH  s      rJ   set_grad_checkpointingz(EfficientFormerV2.set_grad_checkpointinga  s     	*A#)A 	*rK   r   c                 2    | j                   | j                  fS rM   r3  r5  )rC   s    rJ   get_classifierz EfficientFormerV2.get_classifierf  s    yy$..((rK   r"  r#  c                 (   || _         ||| _        |dkD  r t        j                  | j                  |      nt        j
                         | _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   )r"  r#  rm   r2  r.  r   r3  r5  )rC   r"  r#  s      rJ   reset_classifierz"EfficientFormerV2.reset_classifierj  sq    &"*DALqBIId//=VXVaVaVc	FQTUo4#4#4kB[][f[f[hrK   c                     || _         y rM   )r8  )rC   r\  s     rJ   set_distilled_trainingz(EfficientFormerV2.set_distilled_trainingq  s
    "(rK   rO   indicesr0  
stop_early
output_fmtintermediates_onlyc                 &   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        | j                        dz
  }
t        j
                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]O  \  }} ||      }||v s||
k(  r'|r| j                  |      n|}|j                  |       ?|j                  |       Q |r|S |
k(  r| j                  |      }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r   N)
r   r&  r-  r%  r   r   r  	enumerater0  r,  )rC   rO   re  r0  rf  rg  rh  intermediatestake_indices	max_indexlast_idxr-  feat_idxrJ  x_inters                  rJ   forward_intermediatesz'EfficientFormerV2.forward_intermediatesu  s   * Y&D(DD&"6s4;;7G"Qi IIaLt{{#a'99!!#:[[F[[)a-0F(0 	,OHeaA<'x'.2diilG!((1!((+	,   x		!A-rK   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rQ   )r   r&  r-  rm   r   r0  rb  )rC   re  rs  rt  rm  rn  s         rJ   prune_intermediate_layersz+EfficientFormerV2.prune_intermediate_layers  s]     #7s4;;7G"Qikk.9q=1DI!!!R(rK   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rM   )r%  r-  r0  rN   s     rJ   forward_featuresz"EfficientFormerV2.forward_features  s.    IIaLKKNIIaLrK   
pre_logitsc                 6   | j                   dk(  r|j                  d      }| j                  |      }|r|S | j                  |      | j	                  |      }}| j
                  r.| j                  r"t        j                  j                         s||fS ||z   dz  S )Nr  )r6   r3   r   r6   )
r#  meanr1  r3  r5  r8  r   r   r   r  )rC   rO   ry  x_dists       rJ   forward_headzEfficientFormerV2.forward_head  s    u$6"ANN1HIIaL$.."36""t}}UYY=S=S=Uf9 J!##rK   c                 J    | j                  |      }| j                  |      }|S rM   )rx  r}  rN   s     rJ   rP   zEfficientFormerV2.forward  s'    !!!$a rK   )r3   r&   r  NNr0   rR   r   gelu  r   r   r   r   r   Tr  r   rM   )NFFrj  F)r   FT)rT   rU   rV   r@   r7  r   r   ignorerV  rZ  r]  rm   Moduler`  rr   r   r   rb  rd  r   r   r   boolr   rr  rv  rx  r}  rP   rW   rX   s   @rJ   r   r     s    $#'%L(^- YYR R YY  YY* * YY)		 ) )iC ihsm i YY) ) 8<$$',1 ||1  eCcN341  	1 
 1  1  !%1  
tELL!5tELL7I)I#JJ	K1 j ./$#	3S	>*  	 $$ $rK   c                 2    | ddd dddt         t        ddd|S )	Nr  )r3   r&   r&   Tgffffff?bicubicr_  zstem.conv1.conv)urlr"  
input_size	pool_sizefixed_input_sizecrop_pctinterpolationr{  rM  
classifier
first_convr	   )r  kwargss     rJ   _cfgr    s4    =tae)%.B+;L  rK   ztimm/)	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kc                 h    |j                  dd      }t        t        | |fdt        d|      i|}|S )Nout_indices)r   r   r6   r3   feature_cfgT)flatten_sequentialr  )popr   r   r+  )variant
pretrainedr  r  models        rJ   _create_efficientformerv2r    sC    **]L9K 7JDkJ E LrK   r   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr,   r6   r   r9  r<  r  rB  r>  r  )efficientformerv2_s0r+  EfficientFormer_depthEfficientFormer_width EfficientFormer_expansion_ratiosr  r  r  
model_argss      rJ   r  r    L    $T*(.3D9J %q
qVZ[eVpioVpqqrK   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr+   r6   r   r  r  )efficientformerv2_s1r  r  s      rJ   r  r    r  rK   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr*   r0   rL  r  r  )efficientformerv2_s2r  r  s      rJ   r  r    sL    $T*(.3D9J %q
qVZ[eVpioVpqqrK   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr)   r5   g?r  r  )efficientformerv2_lr  r  s      rJ   r  r    sL    $S)(-3C8J %pzpUYZdUohnUopprK   )rQ   r  )<r   rj   	functoolsr   typingr   r   r   r   r   r   torch.nnrm   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r  r  r  r8   rZ   r   r   r   r   r   r   r  r  r  r   r  default_cfgsr  r  r  r  r   rK   rJ   <module>r     s      5 5   A d d M M * + ' < 
 



	  



	  
_
P
A
2	$  ryy DZ%((// Zzuxx PEHHOO Pf* *Z&bii &R<299 </RYY /d
BMM 
ARYY AHP		 Pf %+/, ,0, ,0, +/+&   r8I r r r8I r r r8I r r q7H q qrK   