
    kh                     N   d Z ddlmZ ddlmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ d	d
lmZ d	dl m!Z! d	dl"m#Z# d	dl$m%Z%m&Z& ddgZ'e G d d             Z( G d dejR                        Z*dde+de,de
fdZ- G d dejR                        Z.e! G d dejR                               Z/	 	 	 	 dde0de0de+d ee
   d!ee
   d"e1deejd                  e0ee+e	f   f   fd#Z3 e4dd$d%d&d'd(d)d*d+d,d-d.d/d0d12      Z5 G d3 dejR                        Z6	 	 	 	 	 dd4ee0d5f   d6ee0d5f   d7ee0   d!e+d8ee+   d9eee+e	f      de(fd:Z7dd4ee0d5f   d6ee0d5f   de(fd;Z8	 	 	 	 	 	 	 dd4ee0d5f   d6ee0d5f   d7e0d=e,d>e,d!e+d8e+d9eee+e	f      de(fd?Z9	 	 	 d	d4ee0d5f   d6ee0d5f   d!e+d@e1de(f
dAZ: e4d
i dB e:dCD      dE e:dFD      dG e:dHD      dI e:dJD      dK e:dLD      dM e:dND      dO e:dPD      dQ e9dCD      dR e9dFD      dS e9dHD      dT e9dJD      dU e9dLD      dV e9dND      dW e9dPD      dX e9dYD      dZ e9dCd[d\d] e4d]d^_      d`a      db e9dCd[d\d]dc e4       d`d      de e9dFdfd\d]dc e4       d`d      dg e9dHdfd\d]dc e4       d`d      dh e9dJdfd\d]dc e4       d`d      di e8djD      dk e8dlD      dm e8dndop      dq e8drdsp      dt e8dudvp      dw e8dxdyp      dz e7d{D      d| e7d}D      d~ e7dD      d e7d{d< e4d            d e7d}d< e4d            d e7dd< e4d            d e7d{dc e4             d e7d}dc e4             d e7ddc e4             d e9ddd[d^d] e4d]d^_      d`      Z;dde+de1de	de6fdZ<dde+de	dee+e	f   fdZ= e%i d e=ddddddd      d e=ddddddd      d e=ddddddd      d e=ddddddd      d e=ddddddd      d e=ddddddd      d e=ddddddd      dQ e=dddd      dR e=dddd      dS e=dddd      dT e=dddd      dU e=dddd      dV e=dddd      dW e=dddd      dX e=dddd      d e=ddddddī      d e=ddddddī      i d e=ddddddī      d e=ddddddī      dh e=ddddd̫      di e=dddddͬΫ      d e=ddddddͬѫ      dm e=dddddͬΫ      dq e=dddddͬΫ      dt e=dddddͬΫ      dw e=dddddͬΫ      dz e=ddͬ֫      d e=dddddddͬ٫      d~ e=ddͬ֫      d e=ddͬ֫      d e=ddͬ֫      d e=ddͬ֫      d e=ddͬ֫      d e=ddͬ֫       e=ddͬ֫       e=ddddddݬޫ      dߜ      Z>e&dde1de	de6fd       Z?e&dde1de	de6fd       Z@e&dde1de	de6fd       ZAe&dde1de	de6fd       ZBe&dde1de	de6fd       ZCe&dde1de	de6fd       ZDe&dde1de	de6fd       ZEe&dde1de	de6fd       ZFe&dde1de	de6fd       ZGe&dde1de	de6fd       ZHe&dde1de	de6fd       ZIe&dde1de	de6fd       ZJe&dde1de	de6fd       ZKe&dde1de	de6fd       ZLe&dde1de	de6fd       ZMe&dde1de	de6fd       ZNe&dde1de	de6fd       ZOe&dde1de	de6fd       ZPe&dde1de	de6fd       ZQe&dde1de	de6fd       ZRe&dde1de	de6fd       ZSe&dde1de	de6fd       ZTe&dde1de	de6fd       ZUe&dde1de	de6fd       ZVe&dde1de	de6fd       ZWe&dde1de	de6fd       ZXe&dde1de	de6fd       ZYe&dde1de	de6fd       ZZe&dde1de	de6fd       Z[e&dde1de	de6fd       Z\e&dde1de	de6fd       Z]e&dde1de	de6fd       Z^e&dde1de	de6fd        Z_e&dde1de	de6fd       Z`e&dde1de	de6fd       Zae&dde1de	de6fd       Zby(  a   Normalization Free Nets. NFNet, NF-RegNet, NF-ResNet (pre-activation) Models

Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
    - https://arxiv.org/abs/2101.08692

Paper: `High-Performance Large-Scale Image Recognition Without Normalization`
    - https://arxiv.org/abs/2102.06171

Official Deepmind JAX code: https://github.com/deepmind/deepmind-research/tree/master/nfnets

Status:
* These models are a work in progress, experiments ongoing.
* Pretrained weights for two models so far, more to come.
* Model details updated to closer match official JAX code now that it's released
* NF-ResNet, NF-RegNet-B, and NFNet-F models supported

Hacked together by / copyright Ross Wightman, 2021.
    )OrderedDict)	dataclassreplace)partial)AnyCallableDictOptionalTupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	ClassifierHeadDropPathAvgPool2dSameScaledStdConv2dScaledStdConv2dSameget_act_layer
get_act_fnget_attnmake_divisible   )build_model_with_cfg)register_notrace_module)checkpoint_seq)generate_default_cfgsregister_modelNormFreeNetNfCfgc                   n   e Zd ZU dZeeeeef   ed<   eeeeef   ed<   dZeed<   dZ	e
ed<   dZee   ed	<   dZee   ed
<   dZee
   ed<   dZeee
ef      ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Ze
ed!<   y)"r   z.Configuration for Normalization-Free Networks.depthschannelsg?alpha3x3	stem_typeNstem_chs
group_size
attn_layerattn_kwargs       @	attn_gain      ?width_factor      ?bottle_ratior   num_features   ch_divFreg
extra_convgamma_in_actsame_paddinggh㈵>std_conv_epsskipinitzero_init_fcsilu	act_layer) __name__
__module____qualname____doc__r   int__annotations__r#   floatr%   strr&   r
   r'   r(   r)   r	   r   r+   r-   r/   r0   r2   r3   boolr4   r5   r6   r7   r8   r9   r;        M/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/nfnet.pyr   r   &   s   8#sC$%%Cc3&''E5Is"Hhsm" $J$ $J$,0K$sCx.)0IuL%L%L#FCOCJL$L$L%HdL$IsrF   c                   j     e Zd ZdZd	dededef fdZdej                  dej                  fdZ
 xZS )
GammaActz.Activation function with gamma scaling factor.act_typegammainplacec                 ^    t         |           t        |      | _        || _        || _        y)zInitialize GammaAct.

        Args:
            act_type: Type of activation function.
            gamma: Scaling factor for activation output.
            inplace: Whether to perform activation in-place.
        N)super__init__r   act_fnrK   rL   )selfrJ   rK   rL   	__class__s       rG   rO   zGammaAct.__init__C   s*     	 *
rF   xreturnc                 n    | j                  || j                        j                  | j                        S )zzForward pass.

        Args:
            x: Input tensor.

        Returns:
            Scaled activation output.
        rL   )rP   rL   mul_rK   rQ   rS   s     rG   forwardzGammaAct.forwardP   s*     {{1dll{388DDrF   )relur,   F)r<   r=   r>   r?   rC   rB   rD   rO   torchTensorrY   __classcell__rR   s   @rG   rI   rI   @   s>    8 e D 	E 	E%,, 	ErF   rI   r,   rJ   rK   rT   c                 2     ddt         dt        f fd}|S )zCreate activation function factory with gamma scaling.

    Args:
        act_type: Type of activation function.
        gamma: Scaling factor for activation output.

    Returns:
        Activation function factory.
    rL   rT   c                      t        |       S )N)rK   rL   )rI   )rL   rJ   rK   s    rG   _createzact_with_gamma.<locals>._createf   s    w??rF   F)rD   rI   )rJ   rK   ra   s   `` rG   act_with_gammarc   \   s    @ @( @NrF   c                        e Zd ZdZdddefdededededee   d	ef fd
Zde	j                  de	j                  fdZ xZS )DownsampleAvgzEAvgPool downsampling as in 'D' ResNet variants with dilation support.r   Nin_chsout_chsstridedilationfirst_dilation
conv_layerc                    t         t        |           |dk(  r|nd}|dkD  s|dkD  r2|dk(  r|dkD  rt        nt        j
                  } |d|dd      | _        nt	        j                         | _         |||dd      | _        y)a7  Initialize DownsampleAvg.

        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for downsampling.
            dilation: Dilation rate.
            first_dilation: First dilation rate (unused).
            conv_layer: Convolution layer type.
        r      TF)	ceil_modecount_include_pad)rh   N)	rN   re   rO   r   nn	AvgPool2dpoolIdentityconv)
rQ   rf   rg   rh   ri   rj   rk   
avg_strideavg_pool_fnrR   s
            rG   rO   zDownsampleAvg.__init__n   sx    & 	mT+-'1}V!
A:A+5?x!|-QSQ]Q]K#AzTUZ[DIDIvw!<	rF   rS   rT   c                 B    | j                  | j                  |            S )ztForward pass.

        Args:
            x: Input tensor.

        Returns:
            Downsampled tensor.
        )rt   rr   rX   s     rG   rY   zDownsampleAvg.forward   s     yy1&&rF   )r<   r=   r>   r?   r   r@   r
   r   rO   r[   r\   rY   r]   r^   s   @rG   re   re   k   sr    O ,0#2== = 	=
 = %SM= !=8	' 	'%,, 	'rF   re   c            %            e Zd ZdZddddddddddddddded	fd
edee   dededee   dedededee   dededededee	   dedee	   de	def$ fdZ
dej                  dej                  fdZ xZS ) NormFreeBlockz-Normalization-Free pre-activation block.
    Nr   r,         ?TFr*           rf   rg   rh   ri   rj   r#   betar/   r'   r2   r3   r4   r8   r(   r+   r;   rk   drop_path_ratec                 \   t         |           |xs |}|xs |}t        |r||z  n||z  |
      }|	sdn||	z  }|	r|	|
z  dk(  r|	|z  }|| _        || _        || _        ||k7  s
|dk7  s||k7  rt        ||||||      | _        nd| _         |       | _         |||d      | _	         |d      | _
         |||d|||      | _        |r" |d      | _         |||dd||      | _        nd| _        d| _        |r| ||      | _        nd| _         |       | _         |||d|rd	nd
      | _        |s| ||      | _        nd| _        |dkD  rt%        |      nt'        j(                         | _        |r.t'        j,                  t/        j0                  d
            | _        yd| _        y)a  Initialize NormFreeBlock.

        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for convolution.
            dilation: Dilation rate.
            first_dilation: First dilation rate.
            alpha: Alpha scaling factor for residual.
            beta: Beta scaling factor for pre-activation.
            bottle_ratio: Bottleneck ratio.
            group_size: Group convolution size.
            ch_div: Channel divisor for rounding.
            reg: Use RegNet-style configuration.
            extra_conv: Add extra 3x3 convolution.
            skipinit: Use skipinit initialization.
            attn_layer: Attention layer type.
            attn_gain: Attention gain factor.
            act_layer: Activation layer type.
            conv_layer: Convolution layer type.
            drop_path_rate: Stochastic depth drop rate.
        r   r   )rh   ri   rj   rk   NTrV      )rh   ri   groupsr,   r{   )	gain_init)rN   rO   r   r#   r|   r+   re   
downsampleact1conv1act2conv2act2bconv2battnact3conv3	attn_lastr   rp   rs   	drop_path	Parameterr[   tensorskipinit_gain)rQ   rf   rg   rh   ri   rj   r#   r|   r/   r'   r2   r3   r4   r8   r(   r+   r;   rk   r}   mid_chsr   rR   s                        rG   rO   zNormFreeBlock.__init__   s   V 	'38#V #,!67\CY[ab$'Z*?*v-2 6)G
	"W!x>/I+!-%DO #DOK	3
d+	!F^djk
"40DJ$WgqX^deDKDJDK:)"7+DIDIK	!XrSUV
z-'0DN!DN5Ca5G.1R[[]?GR\\%,,r*:;TrF   rS   rT   c                    | j                  |      | j                  z  }|}| j                  | j                  |      }| j                  |      }| j	                  | j                  |            }| j                   | j                  | j                  |            }| j                  | j                  | j                  |      z  }| j                  | j                  |            }| j                  | j                  | j                  |      z  }| j                  |      }| j                  |j                  | j                         || j                   z  |z   }|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output tensor.
        )r   r|   r   r   r   r   r   r   r   r+   r   r   r   r   r   rW   r#   )rQ   rS   outshortcuts       rG   rY   zNormFreeBlock.forward   s    iilTYY& ??&s+H jjojj3(;;"++djjo.C99 ..499S>1Cjj3(>>%..4>>##66CnnS!)HHT''(DJJ)
rF   )r<   r=   r>   r?   r   r@   r
   rB   rD   r   rO   r[   r\   rY   r]   r^   s   @rG   ry   ry      sL    &*,0"&(,$"-1",0#2$&'XRXR c]XR 	XR
 XR %SMXR XR XR  XR !XR XR XR XR XR !*XR  !XR"  )#XR$ !%XR& "'XRt %,, rF   ry    rf   rg   r%   rk   r;   preact_featurec                 |   d}t        |dd      }t               }|dv sJ d|v rd|v r+d|vsJ |dz  |d	z  |dz  |f}	d
}
d	}t        |dz  dd      }n1d|v rd|z  dz  |dz  |f}	n|dz  |dz  |f}	d}
t        |dz  dd      }t        |	      dz
  }t        t	        |	|
            D ]6  \  }\  }} || |d|      |d|dz    <   ||k7  r |d      |d|dz    <   |} 8 n#d|v r || |dd      |d<   n || |dd      |d<   d|v rt        j                  ddd      |d<   d	}t        j                  |      ||fS )a  Create stem module for NFNet models.

    Args:
        in_chs: Input channels.
        out_chs: Output channels.
        stem_type: Type of stem ('', 'deep', 'deep_tiered', 'deep_quad', '3x3', '7x7', etc.).
        conv_layer: Convolution layer type.
        act_layer: Activation layer type.
        preact_feature: Use pre-activation feature.

    Returns:
        Tuple of (stem_module, stem_stride, stem_feature_info).
    rm   	stem.convnum_chs	reductionmodule)	r   deepdeep_tiered	deep_quadr$   7x7	deep_pool3x3_pool7x7_poolr   quadrr   r1      )rm   r   r   rm   z
stem.conv3tieredr   )rm   r   r   z
stem.conv2r   )kernel_sizerh   rt   TrV   actr$      )rh   padding)dictr   len	enumerateziprp   	MaxPool2d
Sequential)rf   rg   r%   rk   r;   r   stem_stridestem_featurestemr&   strideslast_idxicss                  rG   create_stemr     s   * K1[IL=DssssY***1glGqL'JH"GK1,WL9$K1,glGD#qL'Q,@G1,WLx=1$"3x#9: 	IAv1#-faQq#QD4Aw H}&/&=s1q5']#F		
 
)	!&'qKV "&'qKV||Aa;V==\99rF   g   `U?g   yX?g   \9?g   `aK?g   ?g    ?g    `l?g   `i?g   |?g    7@g   -?g   @g   `?g   ?)identityceluelugelu
leaky_relulog_sigmoidlog_softmaxrZ   relu6selusigmoidr:   softsignsoftplustanhc                        e Zd ZdZ	 	 	 	 	 	 ddededededededed	ef fd
Z	e
j                  j                  ddedeeef   fd       Ze
j                  j                  ddeddfd       Ze
j                  j                  dej$                  fd       Zddedee   ddfdZde
j,                  de
j,                  fdZdde
j,                  dede
j,                  fdZde
j,                  de
j,                  fdZ xZS )r   a*   Normalization-Free Network

    As described in :
    `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
        - https://arxiv.org/abs/2101.08692
    and
    `High-Performance Large-Scale Image Recognition Without Normalization` - https://arxiv.org/abs/2102.06171

    This model aims to cover both the NFRegNet-Bx models as detailed in the paper's code snippets and
    the (preact) ResNet models described earlier in the paper.

    There are a few differences:
        * channels are rounded to be divisible by 8 by default (keep tensor core kernels happy),
            this changes channel dim and param counts slightly from the paper models
        * activation correcting gamma constants are moved into the ScaledStdConv as it has less performance
            impact in PyTorch when done with the weight scaling there. This likely wasn't a concern in the JAX impl.
        * a config option `gamma_in_act` can be enabled to not apply gamma in StdConv as described above, but
            apply it in each activation. This is slightly slower, numerically different, but matches official impl.
        * skipinit is disabled by default, it seems to have a rather drastic impact on GPU memory use and throughput
            for what it is/does. Approx 8-10% throughput loss.
    cfgnum_classesin_chansglobal_pooloutput_stride	drop_rater}   kwargsc                 (   t          |           || _        || _        d| _        t        |fi |}|j                  t        v sJ d|j                   d       |j                  rt        nt        }	|j                  r@t        |j                  t        |j                           }
t        |	|j                        }	n>t        |j                        }
t        |	t        |j                     |j                        }	|j                   r)t        t#        |j                         fi |j$                  nd}t'        |j(                  xs |j*                  d   |j,                  z  |j.                        }t1        |||j2                  |	|
	      \  | _        }}|g| _        t9        j:                  d|t=        |j>                              jA                  |j>                        D cg c]  }|jC                          }}|}|}d
}d}g }tE        |j>                        D ]  \  }}|dk(  r|dkD  rd
nd}||k\  r|d
kD  r||z  }d
}||z  }|dv rd
nd}g }tG        |j>                  |         D ]  }|dk(  xr |dk(  }t'        |j*                  |   |j,                  z  |j.                        }|tI        d-i d|d|d|jJ                  dd|dz  z  d|dk(  r|nd
d|d|d|jL                  d|jN                  r|rdn|jP                  d|j.                  d|jN                  d|jR                  d|jT                  d|d|jV                  d|
d|	d ||   |   gz  }|dk(  rd}||jJ                  dz  z  }|}|} | xj6                  tY        ||d!| "      gz  c_        |t[        j\                  | gz  } t[        j\                  | | _/        |j`                  rqt'        |j,                  |j`                  z  |j.                        | _0         |	|| j`                  d
      | _1        tY        | j`                  |d#"      | j6                  d$<   n || _0        t[        jd                         | _1         |
|j`                  dkD  %      | _3        | j`                  | _4        tk        | j`                  ||| j                  &      | _6        | jo                         D ]:  \  }}d'|v rtq        |tZ        jr                        r|jt                  r*tZ        jv                  jy                  |jz                         n+tZ        jv                  j}                  |jz                  d(d)       |j~                  tZ        jv                  jy                  |j~                         tq        |tZ        j                        stZ        jv                  j                  |jz                  d*d+,       |j~                  tZ        jv                  jy                  |j~                         = yc c}w ).a  
        Args:
            cfg: Model architecture configuration.
            num_classes: Number of classifier classes.
            in_chans: Number of input channels.
            global_pool: Global pooling type.
            output_stride: Output stride of network, one of (8, 16, 32).
            drop_rate: Dropout rate.
            drop_path_rate: Stochastic depth drop-path rate.
            **kwargs: Extra kwargs overlayed onto cfg.
        Fz3Please add non-linearity constants for activation (z).)rK   )eps)rK   r   Nr   )rk   r;   r   r,   rm   )r   rm   rf   rg   r#   r|   r.   rh   ri   rj   r'   r/   r2   r3   r4   r8   r(   r+   r;   rk   r}   zstages.r   
final_convrV   )	pool_typer   fcr{   g{Gz?fan_inlinear)modenonlinearityrE   )BrN   rO   r   r   grad_checkpointingr   r;   _nonlin_gammar6   r   r   r5   rc   r   r7   r   r(   r   r)   r   r&   r"   r-   r2   r   r%   r   feature_infor[   linspacesumr!   splittolistr   rangery   r#   r'   r3   r/   r4   r8   r+   r   rp   r   stagesr0   r   rs   	final_acthead_hidden_sizer   headnamed_modules
isinstanceLinearr9   initzeros_weightnormal_biasConv2dkaiming_normal_)!rQ   r   r   r   r   r   r   r}   r   rk   r;   r(   r&   r   	stem_featrS   drop_path_ratesprev_chs
net_strideri   expected_varr   	stage_idxstage_depthrh   rj   blocks	block_idxfirst_blockrg   nmrR   s!                                   rG   rO   zNormFreeNet.__init__}  s   , 	&""'c$V$}}-v1dehererdssu/vv-,/,<,<(/
&s}}M#--<XYI 1A1ABJ%cmm4I =3OUXUeUefJMP^^WXcnn5IIae
!3<<#B3<<?cFVFV"VX[XbXbc,7MM!-
)	;	 'K/4~~aQTUXU_U_Q`/a/g/ghkhrhr/st!188:tt 
&/

&; %	/"I{#q.[1_Q!F]*vzF"& J"*f"4Q!NF"3::i#89 #	'1n?a(i)@3CSCS)SUXU_U_`= #-4)) lc11 &/!^6	
 & $2  #~~ (+ww;CDTDT ::   #~~ !\\  * "mm (   *!" $39#=i#H#  & >#%L		Q.!)"5#6 $x:X_`i_jVk"l!mmr}}f-..FK%	/L mmV, .s/?/?#BRBR/RTWT^T^ _D(43D3DaHDO$(1B1Bjcm$oDb! (D kkmDO"3+;+;a+?@ $ 1 1"!nn	
	 &&( 	+DAqqyZ2995##GGNN188,GGOOAHHb#666%GGNN166*Aryy)''xh'W66%GGNN166*	+A us   XcoarserT   c                 0    t        d|rdnddfdg      }|S )z"Group parameters for optimization.z^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+)N)z^final_conv)i )r   r   )r   )rQ   r   matchers      rG   group_matcherzNormFreeNet.group_matcher  s.     &,"2JDQ*
 rF   enableNc                     || _         y)z)Enable or disable gradient checkpointing.N)r   )rQ   r   s     rG   set_grad_checkpointingz"NormFreeNet.set_grad_checkpointing  s     #)rF   c                 .    | j                   j                  S )zGet the classifier head.)r   r   )rQ   s    rG   get_classifierzNormFreeNet.get_classifier  s     yy||rF   c                 J    || _         | j                  j                  ||       y)zReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        N)r   r   reset)rQ   r   r   s      rG   reset_classifierzNormFreeNet.reset_classifier  s     '		[1rF   rS   c                    | j                  |      }| j                  r5t        j                  j	                         st        | j                  |      }n| j                  |      }| j                  |      }| j                  |      }|S )zForward pass through feature extraction layers.

        Args:
            x: Input tensor.

        Returns:
            Feature tensor.
        )	r   r   r[   jitis_scriptingr   r   r   r   rX   s     rG   forward_featureszNormFreeNet.forward_features  sg     IIaL""599+A+A+Ct{{A.AAAOOANN1rF   
pre_logitsc                 N    |r| j                  ||      S | j                  |      S )zForward pass through classifier head.

        Args:
            x: Input features.
            pre_logits: Return features before final linear layer.

        Returns:
            Classification logits or features.
        )r  )r   )rQ   rS   r  s      rG   forward_headzNormFreeNet.forward_head,  s(     7Atyyzy2RdiiPQlRrF   c                 J    | j                  |      }| j                  |      }|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output logits.
        )r
  r  rX   s     rG   rY   zNormFreeNet.forward8  s)     !!!$a rF   )  r   avg    r{   r{   rb   )T)N)r<   r=   r>   r?   r   r@   rC   rB   r   rO   r[   r  ignorerD   r	   r   r  rp   Moduler  r
   r  r\   r
  r  rY   r]   r^   s   @rG   r   r   g  s}   0  $$!#!$&{+{+ {+ 	{+
 {+ {+ {+ "{+ {+z YY	D 	T#s(^ 	 	 YY)T )T ) ) YY		  2C 2hsm 2W[ 2%,, 5<< $
Sell 
S 
S 
S %,, rF   r!   .r"   r'   r(   r)   c                 :    |xs i }t        | |ddd||||	      }|S )ar  Create NFNet ResNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        group_size: Group convolution size.
        act_layer: Activation layer type.
        attn_layer: Attention layer type.
        attn_kwargs: Attention layer arguments.

    Returns:
        NFNet configuration.
    r   @   rz   )	r!   r"   r%   r&   r/   r'   r;   r(   r)   )r   )r!   r"   r'   r;   r(   r)   r   s          rG   
_nfres_cfgr  F  s:    * #K

C JrF   c                 ^    d|d   z  dz  }t        d      }t        | |dddd	|d
d|
      }|S )zCreate NFNet RegNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.

    Returns:
        NFNet configuration.
    i   r     r.   rd_ratior$   r1   g      ?g      @Tse)
r!   r"   r%   r'   r-   r/   r0   r3   r(   r)   )r   r   )r!   r"   r0   r)   r   s        rG   
_nfreg_cfgr  j  sP     (2,&#-L$K
!C JrF   r  r/   	feat_multc                 t    t        |d   |z        }||nt        d      }t        | |dd||d||||      }	|	S )a  Create NFNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        group_size: Group convolution size.
        bottle_ratio: Bottleneck ratio.
        feat_mult: Feature multiplier for final layer.
        act_layer: Activation layer type.
        attn_layer: Attention layer type.
        attn_kwargs: Attention layer arguments.

    Returns:
        NFNet configuration.
    r   r.   r  r      T)r!   r"   r%   r&   r'   r/   r4   r0   r;   r(   r)   )r@   r   r   )
r!   r"   r'   r/   r  r;   r(   r)   r0   r   s
             rG   
_nfnet_cfgr     sZ    2 x|i/0L!,!8+dC>PK
!!C JrF   r8   c                 j    t        | |ddddddd|t        |d   dz        |dt        d      	      }|S )
a  Create DeepMind NFNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        act_layer: Activation layer type.
        skipinit: Use skipinit initialization.

    Returns:
        NFNet configuration.
    r   r  r.   Tr   r*   r  r  )r!   r"   r%   r&   r'   r/   r4   r5   r6   r8   r0   r;   r(   r)   )r   r@   r   )r!   r"   r;   r8   r   s        rG   _dm_nfnet_cfgr"    sR    " "+,#&C  JrF   dm_nfnet_f0)r   rm      r   )r!   dm_nfnet_f1)rm   r      r$  dm_nfnet_f2)r   r$     	   dm_nfnet_f3)r   r1      r&  dm_nfnet_f4)   
         dm_nfnet_f5)r$  r&  $   r(  dm_nfnet_f6)r      *      nfnet_f0nfnet_f1nfnet_f2nfnet_f3nfnet_f4nfnet_f5nfnet_f6nfnet_f7)r1      0   r+  nfnet_l0g      ?r  rz   r1   )r  
rd_divisorr:   )r!   r  r'   r/   r)   r;   eca_nfnet_l0eca)r!   r  r'   r/   r(   r)   r;   eca_nfnet_l1rm   eca_nfnet_l2eca_nfnet_l3nf_regnet_b0)r   r   r$  r$  nf_regnet_b1)rm   r   r   r   nf_regnet_b2)rm   r   r1   r1   )8   p      i  )r!   r"   nf_regnet_b3)rm   r-  r)  r)  )rK  r     i  nf_regnet_b4)rm   r$     rQ  )r        ih  nf_regnet_b5)r   r   r4  r4  )P      iP  i  nf_resnet26)rm   rm   rm   rm   nf_resnet50)r   r   r$  r   nf_resnet101)r   r      r   nf_seresnet26g      ?r  )r!   r(   r)   nf_seresnet50nf_seresnet101nf_ecaresnet26nf_ecaresnet50nf_ecaresnet101
test_nfnet)r   r   r   r   )r  r  `   r  )r!   r"   r  r'   r/   r)   r;   variant
pretrainedr   c                 V    t         |    }t        d      }t        t        | |f||d|S )zCreate a NormFreeNet model.

    Args:
        variant: Model variant name.
        pretrained: Load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        NormFreeNet model instance.
    T)flatten_sequential)	model_cfgfeature_cfg)
model_cfgsr   r   r   )rc  rd  r   rg  rh  s        rG   _create_normfreenetrj    sE     7#I$/K   rF   urlc                 0    | dddddt         t        ddd
|S )	zCreate default configuration dictionary.

    Args:
        url: Model weight URL.
        **kwargs: Additional configuration options.

    Returns:
        Configuration dictionary.
    r  r      rn  r   r   ?bicubicz
stem.conv1zhead.fc)
rk  r   
input_size	pool_sizecrop_pctinterpolationmeanstd
first_conv
classifierr   )rk  r   s     rG   _dcfgrz  ,  s2     =v)%.B")  rF   zdm_nfnet_f0.dm_in1kztimm/zmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f0-604f9c3a.pth)r$  r$  )r      r{  )r      r|  rp  squash)	hf_hub_idrk  rs  rr  test_input_sizert  	crop_modezdm_nfnet_f1.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f1-fc540f82.pthro  rm  )r   @  r  gQ?zdm_nfnet_f2.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f2-89875923.pth)r1   r1   )r   `  r  gq=
ףp?zdm_nfnet_f3.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f3-d74ab3aa.pth)r.  r.  )r     r  gGz?zdm_nfnet_f4.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f4-0ac5b10b.pth)r&  r&  )r     r  )r      r  g;On?zdm_nfnet_f5.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f5-ecb20ab1.pth)   r  )r      r  gI+?zdm_nfnet_f6.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f6-e0f12116.pth)r4  r4  )r     r  )r   @  r  gd;O?)rk  rs  rr  r  )r0  r0  )r     r  )r   `  r  znfnet_l0.ra2_in1kzjhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nfnet_l0_ra2-45c6688d.pth)r   rS  rS  )r~  rk  rs  rr  r  test_crop_pctzeca_nfnet_l0.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l0_ra2-e3e9ac50.pthzeca_nfnet_l1.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l1_ra2-7dce93cd.pthzeca_nfnet_l2.ra3_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l2_ra3-da781a61.pth)rQ  rQ  )rk  rs  rr  r  r  r   )rk  rs  rr  r  rx  znf_regnet_b1.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_regnet_b1_256_ra2-ad85cfef.pth)r~  rk  rs  rr  r  rx  )r      r  )r     r  )r)  r)  )r     r  )rk  rx  znf_resnet50.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_resnet50_ra2-9f236009.pth)r~  rk  rs  rr  r  rt  rx  )r.   r.   r.   gffffff?)r      r  )r-  r-  )r~  rv  rw  rt  rr  rs  )r`  ztest_nfnet.r160_in1kc                     t        dd| i|S )z&NFNet-F0 (DeepMind weight compatible).rd  )r#  rj  rd  r   s     rG   r#  r#         NNvNNrF   c                     t        dd| i|S )z&NFNet-F1 (DeepMind weight compatible).rd  )r%  r  r  s     rG   r%  r%    r  rF   c                     t        dd| i|S )z&NFNet-F2 (DeepMind weight compatible).rd  )r'  r  r  s     rG   r'  r'    r  rF   c                     t        dd| i|S )z&NFNet-F3 (DeepMind weight compatible).rd  )r*  r  r  s     rG   r*  r*    r  rF   c                     t        dd| i|S )z&NFNet-F4 (DeepMind weight compatible).rd  )r,  r  r  s     rG   r,  r,    r  rF   c                     t        dd| i|S )z&NFNet-F5 (DeepMind weight compatible).rd  )r1  r  r  s     rG   r1  r1    r  rF   c                     t        dd| i|S )z&NFNet-F6 (DeepMind weight compatible).rd  )r3  r  r  s     rG   r3  r3    r  rF   c                     t        dd| i|S )z	NFNet-F0.rd  )r7  r  r  s     rG   r7  r7         KjKFKKrF   c                     t        dd| i|S )z	NFNet-F1.rd  )r8  r  r  s     rG   r8  r8    r  rF   c                     t        dd| i|S )z	NFNet-F2.rd  )r9  r  r  s     rG   r9  r9    r  rF   c                     t        dd| i|S )z	NFNet-F3.rd  )r:  r  r  s     rG   r:  r:    r  rF   c                     t        dd| i|S )z	NFNet-F4.rd  )r;  r  r  s     rG   r;  r;    r  rF   c                     t        dd| i|S )z	NFNet-F5.rd  )r<  r  r  s     rG   r<  r<    r  rF   c                     t        dd| i|S )z	NFNet-F6.rd  )r=  r  r  s     rG   r=  r=    r  rF   c                     t        dd| i|S )z	NFNet-F7.rd  )r>  r  r  s     rG   r>  r>    r  rF   c                     t        dd| i|S )zNFNet-L0b w/ SiLU.

    My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & SE ratio
    rd  )rA  r  r  s     rG   rA  rA    s     KjKFKKrF   c                     t        dd| i|S )zECA-NFNet-L0 w/ SiLU.

    My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rd  )rC  r  r  s     rG   rC  rC         O*OOOrF   c                     t        dd| i|S )zECA-NFNet-L1 w/ SiLU.

    My experimental 'light' model w/ F1 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rd  )rE  r  r  s     rG   rE  rE    r  rF   c                     t        dd| i|S )zECA-NFNet-L2 w/ SiLU.

    My experimental 'light' model w/ F2 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rd  )rF  r  r  s     rG   rF  rF    r  rF   c                     t        dd| i|S )zECA-NFNet-L3 w/ SiLU.

    My experimental 'light' model w/ F3 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rd  )rG  r  r  s     rG   rG  rG  &  r  rF   c                     t        dd| i|S )z"Normalization-Free RegNet-B0.
    rd  )rH  r  r  s     rG   rH  rH  /       O*OOOrF   c                     t        dd| i|S )z"Normalization-Free RegNet-B1.
    rd  )rI  r  r  s     rG   rI  rI  6  r  rF   c                     t        dd| i|S )z"Normalization-Free RegNet-B2.
    rd  )rJ  r  r  s     rG   rJ  rJ  =  r  rF   c                     t        dd| i|S )z"Normalization-Free RegNet-B3.
    rd  )rN  r  r  s     rG   rN  rN  D  r  rF   c                     t        dd| i|S )z"Normalization-Free RegNet-B4.
    rd  )rP  r  r  s     rG   rP  rP  K  r  rF   c                     t        dd| i|S )z"Normalization-Free RegNet-B5.
    rd  )rT  r  r  s     rG   rT  rT  R  r  rF   c                     t        dd| i|S )z"Normalization-Free ResNet-26.
    rd  )rW  r  r  s     rG   rW  rW  Y       NNvNNrF   c                     t        dd| i|S )z"Normalization-Free ResNet-50.
    rd  )rX  r  r  s     rG   rX  rX  `  r  rF   c                     t        dd| i|S )z#Normalization-Free ResNet-101.
    rd  )rY  r  r  s     rG   rY  rY  g  r  rF   c                     t        dd| i|S )zNormalization-Free SE-ResNet26.rd  )r[  r  r  s     rG   r[  r[  n       P:PPPrF   c                     t        dd| i|S )zNormalization-Free SE-ResNet50.rd  )r\  r  r  s     rG   r\  r\  t  r  rF   c                     t        dd| i|S )z Normalization-Free SE-ResNet101.rd  )r]  r  r  s     rG   r]  r]  z       QJQ&QQrF   c                     t        dd| i|S )z Normalization-Free ECA-ResNet26.rd  )r^  r  r  s     rG   r^  r^    r  rF   c                     t        dd| i|S )z Normalization-Free ECA-ResNet50.rd  )r_  r  r  s     rG   r_  r_    r  rF   c                     t        dd| i|S )z!Normalization-Free ECA-ResNet101.rd  )r`  r  r  s     rG   r`  r`    s     RZR6RRrF   c                     t        dd| i|S )z%Test NFNet model for experimentation.rd  )ra  r  r  s     rG   ra  ra    s     M
MfMMrF   )r,   )r   NNT))r|  r  i   i   NrZ   NN))r@  h      r  )r|  r     r  r  r.   r*   r   r  N)r  r   TrE   rb   )r   )cr?   collectionsr   dataclassesr   r   	functoolsr   typingr   r   r	   r
   r   r[   torch.nnrp   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   _builderr   _features_fxr   _manipulater   	_registryr   r   __all__r   r  rI   rC   rB   rc   re   ry   r@   rD   r   r   r   r   r   r  r  r   r"  ri  rj  rz  default_cfgsr#  r%  r'  r*  r,  r1  r3  r7  r8  r9  r:  r;  r<  r=  r>  rA  rC  rE  rF  rG  rH  rI  rJ  rN  rP  rT  rW  rX  rY  r[  r\  r]  r^  r_  r`  ra  rE   rF   rG   <module>r     sN  $ $ *  7 7   A8 8 8 * 1 ' <'
"   2Eryy E8S   ('BII ('V }BII } }F )-(,#9:9:9: 9: X&	9:
 H%9: 9: 2==#tCH~-.9:z 		""	
			&\")) \B %;$($(04!c3h!S/! SM! 	!
 SM! d38n-! !HuS#X %S/ \a : %;!04(c3h(S/( ( 	(
 ( ( ( d38n-( (Z %;	!c3h!S/! ! 	!
 !H  >\2> ]3> ]3	>
 ^4> _5> _5> _5> |,> }-> }-> ~.> />  /!>" /#>$ /%>* sr$15I+>0 srdf@1>6 btdf@7>< btdf@=>B rdf@C>N <0O>P <0Q>R <:MNS>T <:MNU>V ><OPW>X ><OPY>^ ,/_>` ,/a>b =1c>f LTt]aObcg>h LTt]aObci>j ]tQU_cQdek>n \eQUQWXo>p \eQUQWXq>r mSWSYZs>v &73STcg$15Iw>
B $ # R] .s 3 4S> ( % e&5{]M\^jrte&
 5{]M\`ltve& 5{]M\`ltve& 5{}m^bnvxe&" 5{}m^cowy#e&* 5{}m^cowy+e&2 5{}m^cowy3e&< &]M[=e&@ &]M[Ae&D &]M[Ee&H (}m]Ie&L (}m]Me&P (}m]Qe&T (}m]Ue&X (}m]Ye&^ x]Madf_e&f U{]Madfge&n U{]Madfoe&v U{}mcfhwe&~ E}mcfhe&F E&]MfqsGe&J U A]M^ikKe&R E&]MfqsSe&V E&]MfqsWe&Z E(}mhsu[e&^ E(}mhsu_e&d 5RK8ee&f E{]M\`mxzge&n Eb[9oe&r Urk:se&t Urk:ue&v e{;we&z e{;{e&| e{;}e&~ <!/-6CCe& eP OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 OD OC OK O O
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L
 L L L L L PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P PT PS P[ P P OD OC OK O O OD OC OK O O PT PS P[ P P Qd Qc Qk Q Q
 Qd Qc Qk Q Q
 Rt Rs R{ R R
 Rt Rs R{ R R
 Rt Rs R{ R R
 S S S S S
 N4 N3 N; N NrF   