
    kh[                        d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl
mc mZ ddl	mZ ddlmZmZ ddlmZmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$ dgZ%e G d dejL                               Z' G d dejL                        Z( G d dejL                        Z) G d dejL                        Z* G d dejL                        Z+ G d dejL                        Z,d,dZ-d Z.d-dZ/d.dZ0 e$ e0dd d!       e0dd d!       e0dd"d#d$       e0dd"d#d$       e0dd"d#d$       e0dd#d!      d%      Z1e#d-d&e,fd'       Z2e#d-d&e,fd(       Z3e#d-d&e,fd)       Z4e#d-d&e,fd*       Z5e#d-d&e,fd+       Z6y)/a#   EdgeNeXt

Paper: `EdgeNeXt: Efficiently Amalgamated CNN-Transformer Architecture for Mobile Vision Applications`
 - https://arxiv.org/abs/2206.10589

Original code and weights from https://github.com/mmaaz60/EdgeNeXt

Modifications and additions for timm by / Copyright 2022, Ross Wightman
    N)partial)ListOptionalTupleUnion)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_tf_DropPathLayerNorm2dMlpcreate_conv2dNormMlpClassifierHeadClassifierHead   )build_model_with_cfg)feature_take_indices)register_notrace_module)named_applycheckpoint_seq)register_modelgenerate_default_cfgsEdgeNeXtc                   8     e Zd Zd fd	Zdeeeef   fdZ xZS )PositionalEncodingFourierc                     t         |           t        j                  |dz  |d      | _        dt
        j                  z  | _        || _        || _	        || _
        y )N   r   )kernel_size)super__init__r   Conv2dtoken_projectionmathpiscaletemperature
hidden_dimdim)selfr)   r*   r(   	__class__s       P/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/edgenext.pyr"   z"PositionalEncodingFourier.__init__    sL     "		*q.#1 M[
&$    shapec           
      n   | j                   j                  j                  }| j                   j                  j                  }t	        j
                  |      j                  |t        j                         }|j                  dt        j                        }|j                  dt        j                        }d}||d d dd d d f   |z   z  | j                  z  }||d d d d dd f   |z   z  | j                  z  }t	        j                  | j                  t        j                  |      j                  t        j                        }| j                  dt	        j                  |dd	      z  | j                  z  z  }|d d d d d d d f   |z  }	|d d d d d d d f   |z  }
t	        j                   |	d d d d d d d
d df   j#                         |	d d d d d d dd df   j%                         fd      j'                  d      }	t	        j                   |
d d d d d d d
d df   j#                         |
d d d d d d dd df   j%                         fd      j'                  d      }
t	        j(                  |
|	fd      j+                  d
ddd      }| j                  |j                  |            }|S )N)devicedtyper   )r2   r   ư>)r2   r1   floor)rounding_moder      r*      )r$   weightr1   r2   torchzerostoboolcumsumfloat32r'   aranger)   int64r(   divstacksincosflattencatpermute)r+   r/   r1   r2   inv_masky_embedx_embedepsdim_tpos_xpos_yposs               r-   forwardz!PositionalEncodingFourier.forward(   s^   &&--44%%,,22KK&))uzz)JJ//!5==/9//!5==/9WQQY/#56CWQ23Y/#56CT__EKKORRSXS`S`a  Q5!7)S%SVZVeVe%ef1a&.1a&.1aADqD=!%%'1aADqD=!%%')./118 	 1aADqD=!%%'1aADqD=!%%')./118 	 iiA.66q!QB##CFF5M2
r.   )    i   i'  )__name__
__module____qualname__r"   r   intrR   __classcell__r,   s   @r-   r   r      s    U3S=1 r.   r   c            
       p     e Zd Zdddddd eej
                  d      ej                  df	 fd		Zd
 Z xZ	S )	ConvBlockN   r   Tr7   r3   rM           c                    t         |           |xs |}|dkD  xs ||k7  | _        t        ||||d|      | _         ||      | _        t        |t        ||z        |	      | _        |dkD  r+t        j                  |t        j                  |      z        nd | _        |
dkD  rt        |
      | _        y t        j                         | _        y )Nr   T)r    stride	depthwisebias	act_layerr   r^   )r!   r"   shortcut_after_dwr   conv_dwnormr   rW   mlpr   	Parameterr;   onesgammar   Identity	drop_path)r+   r*   dim_outr    r`   	conv_biasexpand_ratiols_init_value
norm_layerrd   rm   r,   s              r-   r"   zConvBlock.__init__D   s     	.S!'!!=sg~$k&DW`bw'	wL7$: ;yQJWZ[J[R\\-%**W2E"EFae
09B),BKKMr.   c                 :   |}| j                  |      }| j                  r|}|j                  dddd      }| j                  |      }| j	                  |      }| j
                  | j
                  |z  }|j                  dddd      }|| j                  |      z   }|S )Nr   r   r9   r   )rf   re   rI   rg   rh   rk   rm   )r+   xshortcuts      r-   rR   zConvBlock.forward[   s    LLO!!HIIaAq!IIaLHHQK::!

QAIIaAq!t~~a((r.   
rT   rU   rV   r   r   	LayerNormGELUr"   rR   rX   rY   s   @r-   r[   r[   C   s9     r||6ggR.r.   r[   c                   f     e Zd Z	 	 	 	 d fd	Zd Zej                  j                  d        Z xZ	S )CrossCovarianceAttnc                 l   t         |           || _        t        j                  t        j                  |dd            | _        t        j                  ||dz  |      | _	        t        j                  |      | _        t        j                  ||      | _        t        j                  |      | _        y )Nr   r9   )rb   )r!   r"   	num_headsr   ri   r;   rj   r(   LinearqkvDropout	attn_dropproj	proj_drop)r+   r*   r|   qkv_biasr   r   r,   s         r-   r"   zCrossCovarianceAttn.__init__m   s     	"<<

9a(CD99S#'9I.IIc3'	I.r.   c                 H   |j                   \  }}}| j                  |      j                  ||d| j                  d      j	                  ddddd      }|j                  d      \  }}}t        j                  |d      t        j                  |d      j                  dd      z  | j                  z  }	|	j                  d      }	| j                  |	      }	|	|z  }|j	                  dddd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr9   r4   r   r   r7   r   r8   )r/   r~   reshaper|   rI   unbindF	normalize	transposer(   softmaxr   r   r   )
r+   rt   BNCr~   qkvattns
             r-   rR   zCrossCovarianceAttn.forward~   s   ''1ahhqk!!!Q4>>2>FFq!QPQSTU**Q-1a A2&QB)?)I)I"b)QQUYUeUee|||#~~d#AXIIaAq!))!Q2IIaLNN1r.   c                     dhS )Nr(    r+   s    r-   no_weight_decayz#CrossCovarianceAttn.no_weight_decay   s
    r.   )   Fr^   r^   )
rT   rU   rV   r"   rR   r;   jitignorer   rX   rY   s   @r-   rz   rz   l   s9     /"  YY r.   rz   c                   v     e Zd Zddddddd eej
                  d      ej                  dddf fd	Zd	 Z xZ	S )
SplitTransposeBlockr   r   r7   Tr3   r]   r^   c           
         t         |           t        t        t	        j
                  ||z              t        t	        j                  ||z                    }|| _        t        d|dz
        | _        g }t        | j                        D ]!  }|j                  t        ||dd|             # t        j                  |      | _        d | _        |rt!        |      | _         |	|      | _        |dkD  r+t        j$                  |t'        j(                  |      z        nd | _        t-        |||||      | _         |	|d	      | _        t3        |t        ||z        |

      | _        |dkD  r+t        j$                  |t'        j(                  |      z        nd | _        |dkD  rt9        |      | _        y t        j:                         | _        y )Nr   r9   T)r    ra   rb   r8   r   )r|   r   r   r   r3   r]   rc   r^   )r!   r"   maxrW   r%   ceilr5   width
num_scalesrangeappendr   r   
ModuleListconvspos_embdr   norm_xcari   r;   rj   	gamma_xcarz   xcarg   r   rh   rk   r   rl   rm   )r+   r*   r   r|   rp   use_pos_embro   r   rq   rr   rd   rm   r   r   r   r   ir,   s                    r-   r"   zSplitTransposeBlock.__init__   sv     	C		#
"234c$**SJEV:W6XY
aa0t' 	eALLuedYbcd	e]]5)
5#>DM"3JWZ[J[mejjo&EFae&9x9Xac s-	sCs 23yIFSVWFWR\\-%**S/"AB]a
09B),BKKMr.   c           	         |}|j                  t        | j                        dz   d      }g }|d   }t        | j                        D ]+  \  }}|dkD  r|||   z   } ||      }|j	                  |       - |j	                  |d          t        j                  |d      }|j                  \  }}	}
}|j                  ||	|
|z        j                  ddd      }| j                  H| j                  ||
|f      j                  |d|j                  d         j                  ddd      }||z   }|| j                  | j                  | j                  | j                  |            z        z   }|j                  ||
||	      }| j                  |      }| j!                  |      }| j"                  | j"                  |z  }|j                  dddd      }|| j                  |      z   }|S )Nr   r8   r   r4   r   r9   )chunklenr   	enumerater   r;   rH   r/   r   rI   r   rm   r   r   r   rg   rh   rk   )r+   rt   ru   spxspospr   convr   r   HWpos_encodings                r-   rR   zSplitTransposeBlock.forward   s    ggc$**o)qg1V , 	GAt1u#a&[bBJJrN		
 	

3r7IIc1 WW
1aIIaAE"**1a3==$==!Q3;;Ar1771:NVVWXZ[]^_LL At~~q9I0JJKKIIaAq! IIaLHHQK::!

QAIIaAq!t~~a((r.   rv   rY   s   @r-   r   r      sD     r||6gg%RN!r.   r   c                   |     e Zd Zdddddddddddde eej                  d	
      ej                  f fd	Zd Z	 xZ
S )EdgeNeXtStager   r   r7   r\   FT      ?Nr3   r]   c                    t         |           d| _        |s|dk(  rt        j                         | _        n;t        j                   ||      t        j                  ||dd|            | _        |}g }t        |      D ]d  }|||z
  k  r1|j                  t        |||r|dk(  r|nd|||	|||   ||
             n'|j                  t        ||||	|
||||   ||
             |}f t        j                  | | _        y )NFr   r   r    r`   rb   r   )
r*   rn   r`   ro   r    rp   rq   rm   rr   rd   )
r*   r   r|   rp   r   ro   rq   rm   rr   rd   )r!   r"   grad_checkpointingr   rl   
downsample
Sequentialr#   r   r   r[   r   blocks)r+   in_chsout_chsr`   depthnum_global_blocksr|   scalesr    rp   r   downsample_blockro   rq   drop_path_ratesrr   norm_layer_clrd   stage_blocksr   r,   s                       r-   r"   zEdgeNeXtStage.__init__   s   ( 	"'v{ kkmDO mm6"		&'qSDO Fu 	A5,,,##" ')9a1fv!"+$/%1&3"1!"4#0"+ ##'"#)"+%1$/"+&3"1!"4#0"+ F?	@ mm\2r.   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S N)r   r   r;   r   is_scriptingr   r   r+   rt   s     r-   rR   zEdgeNeXtStage.forward#  sS    OOA""599+A+A+Ct{{A.A  AAr.   )rT   rU   rV   r   r   r   rw   rx   r"   rR   rX   rY   s   @r-   r   r      sM    
 " "!",,D9gg%A3Fr.   r   c                   6    e Zd Zddddddddd	d
dddddddej                  ddf fd	Zej                  j                  d(d       Z	ej                  j                  d)d       Z
ej                  j                  dej                  fd       Zd*dedee   fdZ	 	 	 	 	 d+dej"                  deeeee   f      dededededeeej"                     eej"                  eej"                     f   f   fd Z	 	 	 d,deeee   f   d!ed"efd#Zd$ Zd(d%efd&Zd' Z xZS )-r   r9     avg   0   X      r9   r9   	   r9   )r   r   r   r   )r9      r\   r   )r   r   r   r   )r   r   r9   r7   )FTFFr3   r   r7   FTpatchr^   c           
      f   t         |           || _        || _        || _        t        t        d      }t        t        j                  d      }g | _	        |dv sJ |dk(  r@t        j                  t        j                  ||d   dd|       ||d               | _        n@t        j                  t        j                  ||d   ddd|	       ||d               | _        d}g }t        j                  d|t        |            j!                  |      D cg c]  }|j#                          }}|d   }t%        d      D ]  }|d
k(  s|dkD  rd
nd}||z  }|j'                  t)        d#i d|d||   d|d||   d||   d||   d||   d|	|   d|d||   d|
|   d|d|d|d|d|d|       ||   }| xj                  t+        ||d|       gz  c_	         t        j                  | | _        |d   x| _        | _        |r@ || j.                        | _        t5        | j.                  ||| j                         | _        nBt        j8                         | _        t;        | j.                  ||| j                  |!      | _        t=        t        t>        |"      |        y c c}w )$Nr3   r]   )r   overlapr   r   r7   r   r   )r    r`   paddingrb   r   r   r   r   r`   r   r   r|   r   r   rp   r    r   rq   r   ro   rr   r   rd   zstages.)num_chs	reductionmoduler4   )	pool_type	drop_rate)r   r   rr   )head_init_scaler   ) r!   r"   num_classesglobal_poolr   r   r   r   rw   feature_infor   r#   stemr;   linspacesumsplittolistr   r   r   dictstagesnum_featureshead_hidden_sizenorm_prer   headrl   r   r   _init_weights)r+   in_chansr   r   dimsdepthsglobal_block_countskernel_sizesheads	d2_scalesr   rq   r   rp   r   ro   	stem_typehead_norm_firstrd   drop_path_rater   rr   r   curr_strider   rt   dp_ratesr   r   r`   r,   s                                 r-   r"   zEdgeNeXt.__init__-  s   . 	&&"[d3
$70000		(DG19U47#DI
 		(DG1f[de47#DI
 (-q.#f+(V(\(\]c(de1AHHJeeaq 	eA%*a!eQF6!KMM- Q  Qi	
 #6a"8  ( !) !| * )O (N , "2 $ &  ,!" $# ( !WF$vW^_`^aUb"c!dd3	e6 mmV,48H<D1&t'8'89DM&!!%..	DI KKMDM-!!%..%DI 	GM?KTRe fs   J.c                 2    t        d|rd      S g d      S )Nz^stemz^stages\.(\d+)))z^stages\.(\d+)\.downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)z	^norm_pre)i )r   r   )r   )r+   coarses     r-   group_matcherzEdgeNeXt.group_matcher  s'    (.$
 	
5
 	
r.   c                 4    | j                   D ]	  }||_         y r   )r   r   )r+   enabless      r-   set_grad_checkpointingzEdgeNeXt.set_grad_checkpointing  s     	*A#)A 	*r.   returnc                 .    | j                   j                  S r   )r   fcr   s    r-   get_classifierzEdgeNeXt.get_classifier  s    yy||r.   r   r   c                 J    || _         | j                  j                  ||       y r   )r   r   reset)r+   r   r   s      r-   reset_classifierzEdgeNeXt.reset_classifier  s    &		[1r.   rt   indicesrg   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        | j                        dz
  }
t        j
                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]>  \  }} ||      }||v s|r||
k(  r| j                  |      }n|}|j                  |       @ |r|S |
k(  r| j                  |      }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r   N)
r   r   r   r   r;   r   r   r   r   r   )r+   rt   r
  rg   r  r  r  intermediatestake_indices	max_indexlast_idxr   feat_idxstagex_inters                  r-   forward_intermediateszEdgeNeXt.forward_intermediates  s   * Y&D(DD&"6s4;;7G"Qi IIaLt{{#a'99!!#:[[F[[)a-0F(0 	.OHeaA<'H0"mmA.GG$$W-	.   xa A-r.   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r   r   rl   r   r	  )r+   r
  r  r  r  r  s         r-   prune_intermediate_layersz"EdgeNeXt.prune_intermediate_layers  s]     #7s4;;7G"Qikk.9q=1KKMDM!!!R(r.   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   r   s     r-   forward_featureszEdgeNeXt.forward_features  s/    IIaLKKNMM!r.   
pre_logitsc                 N    |r| j                  |d      S | j                  |      S )NT)r  )r   )r+   rt   r  s      r-   forward_headzEdgeNeXt.forward_head  s$    0:tyyty,L		!Lr.   c                 J    | j                  |      }| j                  |      }|S r   )r  r!  r   s     r-   rR   zEdgeNeXt.forward  s'    !!!$a r.   F)Tr   )NFFr  F)r   FT)rT   rU   rV   r   rx   r"   r;   r   r   r   r  Moduler  rW   r   strr	  Tensorr   r   r>   r   r  r  r  r!  rR   rX   rY   s   @r-   r   r   ,  s    " ,%"3"!gg+_SB YY
 
 YY* * YY		  2C 2hsm 2 8<$$',0 ||0  eCcN340  	0 
 0  0  !%0  
tELL!5tELL7I)I#JJ	K0 h ./$#	3S	>*  	 M$ Mr.   r   c                 .   t        | t        j                        rNt        | j                  d       | j
                  *t        j                  j                  | j
                         y y t        | t        j                        rt        | j                  d       t        j                  j                  | j
                         |rPd|v rK| j                  j                  j                  |       | j
                  j                  j                  |       y y y y )Ng{Gz?)stdhead.)
isinstancer   r#   r   r:   rb   initzeros_r}   datamul_)r   namer   s      r-   r   r     s    &"))$C0;;"GGNN6;;' #	FBII	&C0
v{{#GtOMM##O4KK!!/2 $4 
'r.   c                 >   d| v sd| v r| S d| v r| d   } nd| v r| d   } n	d| v r| d   } i }ddl }| j                         D ]  \  }}|j                  dd	      }|j                  d
d|      }|j                  dd|      }|j                  dd      }|j                  dd      }|j                  dd      }|j	                  d      r|j                  dd      }|j
                  dk(  r2d|vr.|j                         |   j                  }|j                  |      }|||<    |S )z Remap FB checkpoints -> timm zhead.norm.weightznorm_pre.weight	model_emamodel
state_dictr   Nzdownsample_layers.0.zstem.zstages.([0-9]+).([0-9]+)zstages.\1.blocks.\2z#downsample_layers.([0-9]+).([0-9]+)zstages.\1.downsample.\2dwconvrf   pwconvzmlp.fcr)  zhead.fc.znorm.rg   z	head.normr   r   )	reitemsreplacesub
startswithndimr3  r/   r   )r3  r2  out_dictr6  r   r   model_shapes          r-   checkpoint_filter_fnr>    sB   Z'+<
+J j ,
	J	(
		#-
H  " 1II,g6FF.0FJFF9;UWXYIIh	*IIh)IIgz*<< 		&+.A66Q;6?**,Q/55K		+&A Or.   c                 N    t        t        | |ft        t        dd      d|}|S )N)r   r   r   r9   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r>  r   )variant
pretrainedkwargsr2  s       r-   _create_edgenextrG  #  s6     ':1\dK 	E
 Lr.   c                 0    | dddddt         t        ddd
|S )	Nr   )r9      rI  )r   r   g?bicubiczstem.0zhead.fc)
urlr   
input_size	pool_sizecrop_pctinterpolationmeanr(  
first_conv
classifierr	   )rK  rF  s     r-   _cfgrS  ,  s0    =v)%.Bi  r.   ztimm/)r9      rT  )	hf_hub_idtest_input_sizetest_crop_pctgffffff?)r9   @  rX  )rU  rN  rV  rW  )zedgenext_xx_small.in1kzedgenext_x_small.in1kzedgenext_small.usi_in1kzedgenext_base.usi_in1kzedgenext_base.in21k_ft_in1kzedgenext_small_rw.sw_in1kr  c           	      L    t        ddd      }t        dd| it        |fi |S )N)r   r      r   r   r7   r7   r7   r7   r   r   r   rE  )edgenext_xx_smallr   rG  rE  rF  
model_argss      r-   r]  r]  Q  s1     \0AVJeJe$zJd]cJdeer.   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr   )rS   @   d      r[  r\  rE  )edgenext_x_smallr^  r_  s      r-   re  re  \  s1     \0B,WJd:djIc\bIcddr.   c           	      J    t        dd      }t        dd| it        |fi |S )Nr   )r   `      i0  r   r   rE  )edgenext_smallr^  r_  s      r-   rj  rj  g  s/     \0BCJbbtJGaZ`Gabbr.   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   )P   rh  rT  iH  ri  rE  )edgenext_baser^  r_  s      r-   rm  rm  r  s/     \0CDJa
ad:F`Y_F`aar.   c           	      P    t        ddddd      }t        dd| it        |fi |S )	Nr   )r   rg  rd  i  TFr   )r   r   r   ro   r   rE  )edgenext_small_rwr^  r_  s      r-   ro  ro  }  s:    "4)EJ eJe$zJd]cJdeer.   )Nr   r#  )r  )7__doc__r%   	functoolsr   typingr   r   r   r   r;   torch.nn.functionalr   
functionalr   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   r   	_registryr   r   __all__r$  r   r[   rz   r   r   r   r   r>  rG  rS  default_cfgsr]  re  rj  rm  ro  r   r.   r-   <module>r~     s     / /     A* * * * + 1 4 <, !		 ! !H&		 &R$")) $NI")) IXJBII JZHryy HV
3> %"%S: "%S:  $}C  #}C $(}C$ "&%S"'& 4 fX f f eH e e c( c c b b b fX f fr.   