
    khY                        d Z ddlmZ ddlmZmZmZmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ  G d de	j:                        Z G d de	j:                        Z G d de	j:                        Z  G d de	j:                        Z! G d de	j:                        Z" G d de	j:                        Z# G d de	j:                        Z$d Z%d9dZ& e e&d       e&d       e&d       e&d       e&d       e&d       e&ddd !       e&ddd !       e&ddd !       e&d       e&dd"d"d#d$%       e&dd&'       e&d(d)d*+      d,      Z'd:d-Z(ed:d.       Z)ed:d/       Z*ed:d0       Z+ed:d1       Z,ed:d2       Z-ed:d3       Z.ed:d4       Z/ed:d5       Z0ed:d6       Z1ed:d7       Z2ed:d8       Z3y);z
MambaOut models for image classification.
Some implementations are modified from:
timm (https://github.com/rwightman/pytorch-image-models),
MetaFormer (https://github.com/sail-sg/metaformer),
InceptionNeXt (https://github.com/sail-sg/inceptionnext)
    )OrderedDict)ListOptionalTupleUnionN)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_DropPath	LayerNorm
LayerScaleClNormMlpClassifierHeadget_act_layer   )build_model_with_cfg)feature_take_indices)checkpoint_seq)register_modelgenerate_default_cfgsc                   N     e Zd ZdZdddej
                  efdef fdZd Z	 xZ
S )StemzV Code modified from InternImage:
        https://github.com/OpenGVLab/InternImage
       `   Tmid_normc                    t         |           t        j                  ||dz  ddd      | _        |r ||dz        nd | _         |       | _        t        j                  |dz  |ddd      | _         ||      | _        y )N   r   r   kernel_sizestridepadding)	super__init__r   Conv2dconv1norm1actconv2norm2)selfin_chsout_chsr   	act_layer
norm_layer	__class__s         P/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/mambaout.pyr$   zStem.__init__   s     	YYqL

 2:Z1-t
;YYqL

  (
    c                 @   | j                  |      }| j                  9|j                  dddd      }| j                  |      }|j                  dddd      }| j                  |      }| j	                  |      }|j                  dddd      }| j                  |      }|S )Nr   r   r   r   )r&   r'   permuter(   r)   r*   r+   xs     r1   forwardzStem.forward6   s    JJqM::!		!Q1%A

1A		!Q1%AHHQKJJqMIIaAq!JJqMr2   )__name__
__module____qualname____doc__r   GELUr   boolr$   r7   __classcell__r0   s   @r1   r   r      s1     !gg ) 	)6
r2   r   c                   ,     e Zd Zddef fd	Zd Z xZS )DownsampleNormFirstr      c                 z    t         |            ||      | _        t        j                  ||ddd      | _        y Nr   r   r   r   )r#   r$   normr   r%   convr+   r,   r-   r/   r0   s       r1   r$   zDownsampleNormFirst.__init__E   s;     	v&	II
	r2   c                     | j                  |      }|j                  dddd      }| j                  |      }|j                  dddd      }|S Nr   r   r   r   )rE   r4   rF   r5   s     r1   r7   zDownsampleNormFirst.forwardU   sI    IIaLIIaAq!IIaLIIaAq!r2   r8   r9   r:   r   r$   r7   r>   r?   s   @r1   rA   rA   C   s      	
 r2   rA   c                   ,     e Zd Zddef fd	Zd Z xZS )
Downsampler   rB   c                 z    t         |           t        j                  ||ddd      | _         ||      | _        y rD   )r#   r$   r   r%   rF   rE   rG   s       r1   r$   zDownsample.__init___   s=     	II
	 w'	r2   c                     |j                  dddd      }| j                  |      }|j                  dddd      }| j                  |      }|S rI   )r4   rF   rE   r5   s     r1   r7   zDownsample.forwardo   sI    IIaAq!IIaLIIaAq!IIaLr2   rJ   r?   s   @r1   rL   rL   ]   s      	( r2   rL   c                   p     e Zd ZdZddej
                  deddf fd	Zdded	e	e
   d
efdZddefdZ xZS )MlpHeadz MLP classification head
      avg           Tc	           
      2   t         
|           |t        ||z        }	nd }	|| _        || _        |	xs || _         ||      | _        |	rSt        j                  t        dt        j                  ||	      fd |       fd ||	      fg            | _        |	| _        n || _        t        j                         | _        |dkD  r"t        j                  | j                  ||      nt        j                         | _        t        j                  |      | _        y )Nfcr(   rE   r   )bias)r#   r$   int	pool_typein_featureshidden_sizerE   r   
Sequentialr   Linear
pre_logitsnum_featuresIdentityrV   Dropouthead_dropout)r+   rZ   num_classesrY   r.   	mlp_ratior/   	drop_raterW   r[   r0   s             r1   r$   zMlpHead.__init__{   s     	 i+56KK"&&5+{+	 mmKryyk:;	$K019 - DO
 !,D +D kkmDOJUXY/"))D--{F_a_j_j_lJJy1r2   rc   rY   reset_otherc                 (   ||| _         |rCt        j                         | _        t        j                         | _        | j
                  | _        |dkD  r&t        j                  | j                  |      | _        y t        j                         | _        y )Nr   )	rY   r   r`   rE   r^   rZ   r_   r]   rV   )r+   rc   rY   rf   s       r1   resetzMlpHead.reset   sh     &DNDI kkmDO $ 0 0D?JQ"))D--{;TVT_T_Tar2   r^   c                     | j                   dk(  r|j                  d      }| j                  |      }| j                  |      }| j	                  |      }|r|S | j                  |      }|S )NrR   )r   r   )rY   meanrE   r^   rb   rV   r+   r6   r^   s      r1   r7   zMlpHead.forward   s`    >>U"vAIIaLOOAa HGGAJr2   )NFF)r8   r9   r:   r;   r   r<   r   r$   rX   r   strr=   rh   r7   r>   r?   s   @r1   rP   rP   w   sX     gg !2Fb b# bTX b	T 	r2   rP   c                   L     e Zd ZdZddddeej                  df fd	Zd Z xZ	S )	GatedConvBlocka   Our implementation of Gated CNN Block: https://arxiv.org/pdf/1612.08083
    Args:
        conv_ratio: control the number of channels to conduct depthwise convolution.
            Conduct convolution on partial channels can improve paraitcal efficiency.
            The idea of partial channels is from ShuffleNet V2 (https://arxiv.org/abs/1807.11164) and
            also used by InceptionNeXt (https://arxiv.org/abs/2303.16900) and FasterNet (https://arxiv.org/abs/2303.03667)
    UUUUUU@         ?NrT   c	                    t         |            ||      | _        t        ||z        }
t	        j
                  ||
dz        | _         |       | _        t        ||z        }|
|
|z
  |f| _        t	        j                  ||||dz  |      | _
        t	        j
                  |
|      | _        |t        |      nt	        j                         | _        |dkD  rt        |      | _        y t	        j                         | _        y )Nr   )r    r"   groupsrT   )r#   r$   rE   rX   r   r]   fc1r(   split_indicesr%   rF   fc2r   r`   lsr   	drop_path)r+   dimexpansion_ratior    
conv_ratiols_init_valuer/   r.   ry   kwargshiddenconv_channelsr0   s               r1   r$   zGatedConvBlock.__init__   s     	sO	_s*+99S&1*-;J,-$f}&<mLII#1$ 
	 99VS)%2%>*S/BKKM09B),BKKMr2   c                    |}| j                  |      }| j                  |      }t        j                  || j                  d      \  }}}|j                  dddd      }| j                  |      }|j                  dddd      }| j                  | j                  |      t        j                  ||fd      z        }| j                  |      }| j                  |      }||z   S )N)rz   r   r   r   r   )rE   ru   torchsplitrv   r4   rF   rw   r(   catrx   ry   )r+   r6   shortcutgics         r1   r7   zGatedConvBlock.forward   s    IIaLHHQK++a!3!3<1aIIaAq!IIaLIIaAq!HHTXXa[599aV#<<=GGAJNN18|r2   )
r8   r9   r:   r;   r   r   r<   r$   r7   r>   r?   s   @r1   ro   ro      s.     " ggR<r2   ro   c            
       l     e Zd Zdddddddeej
                  df
dee   d	ed
edee	   f fdZ
d Z xZS )MambaOutStageNrS   rp   rq   rr    rT   dim_outdepth
downsampler}   c                    t         |           |xs |}d| _        |dk(  rt        |||	      | _        n9|dk(  rt        |||	      | _        n ||k(  sJ t        j                         | _        t        j                  t        |      D cg c]0  }t        ||||||	|
t        |t        t        f      r||   n|      2 c} | _        y c c}w )NFrF   )r/   conv_nf)rz   r{   r    r|   r}   r/   r.   ry   )r#   r$   grad_checkpointingrL   r   rA   r   r`   r\   rangero   
isinstancelisttupleblocks)r+   rz   r   r   r{   r    r|   r   r}   r/   r.   ry   jr0   s                r1   r$   zMambaOutStage.__init__   s     	.S"'(g*MDO9$1#w:VDO'>!> kkmDOmm 5\&
   /'%+%#*4Yu*N)A,T]	&
  &
s   5C
c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S N)r   r   r   jitis_scriptingr   r   r5   s     r1   r7   zMambaOutStage.forward  sS    OOA""599+A+A+Ct{{A.A  AAr2   )r8   r9   r:   r   r   r<   r   rX   rm   floatr$   r7   r>   r?   s   @r1   r   r      sb    
 &*! -1 gg& c]& 	& & $E?&Pr2   r   c                   8    e Zd ZdZdddddeej                  ddd	d
dddddf fd	Zd Ze	j                  j                  d&d       Ze	j                  j                  d'd       Ze	j                  j                  dej                  fd       Zd(dedee   fdZ	 	 	 	 	 d)de	j(                  deeeee   f      dededededeee	j(                     ee	j(                  ee	j(                     f   f   fdZ	 	 	 d*deeee   f   ded efd!Zd" Zd&d#efd$Zd% Z xZS )+MambaOuta<   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels. Default: 3.
        num_classes (int): Number of classes for classification head. Default: 1000.
        depths (list or tuple): Number of blocks at each stage. Default: [3, 3, 9, 3].
        dims (int): Feature dimension at each stage. Default: [96, 192, 384, 576].
        downsample_layers: (list or tuple): Downsampling layers before each stage.
        drop_path_rate (float): Stochastic depth rate. Default: 0.
        output_norm: norm before classifier head. Default: partial(nn.LayerNorm, eps=1e-6).
        head_fn: classification head. Default: nn.Linear.
        head_dropout (float): dropout for MLP classifier. Default: 0.
    r   rQ   rR   r   r   	   r   r        i@  rr   rp   rq   TNrF   rT   defaultc                 (   t         |           || _        || _        d| _        t        |t        t        f      s|g}t        |t        t        f      s|g}t        |      }t        |      }|| _
        g | _        t        ||d   |||      | _        |d   }t        j                  d|t!        |            j#                  |      D cg c]  }|j%                          }}d}d}t'        j(                         | _        t-        |      D ]  }||   }|dk(  s|dkD  rdnd}||z  }t/        ||||   |
||	|dkD  r|nd|||||         }| j*                  j1                  |       |}| xj                  t3        ||d	| 
      gz  c_        |||   z  } |dk(  rt5        |||||      | _        n"t9        ||t;        |dz        |||      | _        || _        | j6                  j<                  | _        | jA                  | jB                         y c c}w )NNHWCr   )r   r.   r/   rS   r   r   r   )rz   r   r   r    r|   r{   r   r}   r/   r.   ry   zstages.)num_chs	reductionmoduler   )rY   re   r/   )r[   rY   r/   re   )"r#   r$   rc   re   
output_fmtr   r   r   r   len	num_stagefeature_infor   stemr   linspacesumr   tolistr   r\   stagesr   r   appenddictrP   headr   rX   r_   head_hidden_sizeapply_init_weights)r+   in_chansrc   global_pooldepthsdimsr/   r.   r|   r{   r    stem_mid_normr}   r   drop_path_ratere   head_fnr   prev_dimr6   dp_ratescurcurr_strider   rz   r!   stager0   s                              r1   r$   zMambaOut.__init__,  s0   & 	&" &4-0XF$u.6D!),	K	"G"!
	 7(-q.#f+(V(\(\]c(de1AHHJeemmoy! 	Aq'C%*a!eQF6!K!Qi'% /)*Q:B+%#"1+E KKu%H$x;Y`ab`cWd"e!ff6!9C+	. i%#%DI 01-%%#DI % $		 6 6

4%%&c fs   	Hc                     t        |t        j                  t        j                  f      rOt	        |j
                  d       |j                  +t        j                  j                  |j                  d       y y y )Ng{Gz?)stdr   )	r   r   r%   r]   r   weightrW   init	constant_)r+   ms     r1   r   zMambaOut._init_weights  sS    a"))RYY/0!((,vv!!!!&&!, " 1r2   c                 2    t        d|rd      S ddg      S )Nz^stemz^stages\.(\d+))z^stages\.(\d+)\.downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)r   r   )r   )r+   coarses     r1   group_matcherzMambaOut.group_matcher  s/    (.$
 	
 685
 	
r2   c                 4    | j                   D ]	  }||_         y r   )r   r   )r+   enabless      r1   set_grad_checkpointingzMambaOut.set_grad_checkpointing  s     	*A#)A 	*r2   returnc                 .    | j                   j                  S r   )r   rV   )r+   s    r1   get_classifierzMambaOut.get_classifier  s    yy||r2   rc   r   c                 J    || _         | j                  j                  ||       y r   )rc   r   rh   )r+   rc   r   s      r1   reset_classifierzMambaOut.reset_classifier  s    &		[1r2   r6   indicesrE   
stop_earlyr   intermediates_onlyc           	         |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
| j                  |      }t        j
                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]#  \  }} ||      }||	v s|j                  |       % |r/|D cg c]$  }|j                  dddd      j                         & }}|r|S ||fS c c}w )	a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWr   z*Output format must be one of NCHW or NHWC.r   Nr   r   r   r   )r   r   r   r   r   r   r   	enumerater   r4   
contiguous)r+   r6   r   rE   r   r   r   channel_firstintermediatestake_indices	max_indexr   feat_idxr   ys                  r1   forward_intermediateszMambaOut.forward_intermediates  s   * --[/[[-"f,"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
 IVWAQYYq!Q2==?WMW  - Xs   <)C/
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r   )r+   r   r   r   r   r   s         r1   prune_intermediate_layersz"MambaOut.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r2   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r5   s     r1   forward_featureszMambaOut.forward_features  s!    IIaLKKNr2   r^   c                 V    |r| j                  ||      }|S | j                  |      }|S )N)r^   )r   rk   s      r1   forward_headzMambaOut.forward_head  s1    3=DIIaJI/ DH99Q<r2   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r5   s     r1   r7   zMambaOut.forward  s'    !!!$a r2   rl   )Tr   )NFFr   F)r   FT)r8   r9   r:   r;   r   r   r<   r$   r   r   r   ignorer   r   Moduler   rX   r   rm   r   Tensorr   r   r=   r   r   r   r   r   r7   r>   r?   s   @r1   r   r     s   $ $ gg#Z'x- YY
 
 YY* * YY		  2C 2hsm 2 8<$$',- ||-  eCcN34-  	- 
 -  -  !%-  
tELL!5tELL7I)I#JJ	K- b ./$#	3S	>*  	
$ r2   r   c                    d| v r| d   } d| v r| S dd l }i }| j                         D ]  \  }}|j                  dd      }|j                  dd|      }|j                  dd	|      }|j	                  d
      r|j                  d
d      }nG|j	                  d      r6|j                  dd      }|j                  dd      }|j                  dd      }|||<    |S )Nmodelzstem.conv1.weightr   zdownsample_layers.0.zstem.zstages.([0-9]+).([0-9]+)zstages.\1.blocks.\2zdownsample_layers.([0-9]+)zstages.\1.downsampleznorm.z
head.norm.zhead.z	head.fc1.zhead.pre_logits.fc.zhead.pre_logits.norm.z	head.fc2.zhead.fc.)reitemsreplacesub
startswith)
state_dictr   r   out_dictkvs         r1   checkpoint_filter_fnr     s    *(
j(H  " 1II,g6FF.0FJFF02I1M<< 		'<0A\\'"		+'<=A		,(?@A		+z2A Or2   c                 2    | ddddddt         t        ddd	|S )
NrQ   )r      r   )r      r   )rq   rq   rr   bicubicz
stem.conv1zhead.fc)urlrc   
input_sizetest_input_size	pool_sizecrop_pctinterpolationrj   r   
first_conv
classifierr	   )r  r~   s     r1   _cfgr
  
  s2    =]y%.B")  r2   ztimm/)	hf_hub_idgffffff?rr   )r  r  test_crop_pct)r   r   r   squash)   r  )r  r  r  	crop_moder  i-.  )r  rc   )r      r  )r   r   r   )   r  )r  r  r  )zmambaout_femto.in1kzmambaout_kobe.in1kzmambaout_tiny.in1kzmambaout_small.in1kzmambaout_base.in1kzmambaout_small_rw.sw_e450_in1kz#mambaout_base_short_rw.sw_e500_in1kz"mambaout_base_tall_rw.sw_e500_in1kz"mambaout_base_wide_rw.sw_e500_in1kz+mambaout_base_plus_rw.sw_e150_in12k_ft_in1kz0mambaout_base_plus_rw.sw_e150_r384_in12k_ft_in1kz#mambaout_base_plus_rw.sw_e150_in12ktest_mambaoutc                 N    t        t        | |ft        t        dd      d|}|S )N)r   r   r   r   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r   r   )variant
pretrainedr~   r   s       r1   _create_mambaoutr  A  s6     ':1\dK 	E Lr2   c           	      J    t        dd      }t        dd| it        |fi |S )Nr   0   r   r   r   r   r   r  )mambaout_femtor   r  r  r~   
model_argss      r1   r  r  L  s-    \0BCJbbtJGaZ`Gabbr2   c           	      R    t        g dg d      }t        dd| it        |fi |S )N)r   r      r   r  r  r  )mambaout_kober   r!  s      r1   r%  r%  R  s-    ]1CDJa
ad:F`Y_F`aar2   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   r   r  r  )mambaout_tinyr   r!  s      r1   r'  r'  W  s-    \0CDJa
ad:F`Y_F`aar2   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   rS      r   r   r  r  )mambaout_smallr   r!  s      r1   r+  r+  ]  s-    ]1DEJbbtJGaZ`Gabbr2   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr)        i   i   r  r  )mambaout_baser   r!  s      r1   r0  r0  c  s-    ]1EFJa
ad:F`Y_F`aar2   c           	      Z    t        g dg ddddd      }t        d	d| it        |fi |S )
Nr)  r   Fr   ư>norm_mlp)r   r   r   r   r}   r   r  )mambaout_small_rwr   r!  s      r1   r4  r4  i  s>     J eJe$zJd]cJdeer2   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )N)r   r      r   r-        @      ?Fr   r2  r3  r   r   r{   r|   r   r   r}   r   r  )mambaout_base_short_rwr   r!  s      r1   r:  r:  v  sE    !	J jjtT^OibhOijjr2   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )Nr   rS      r   r-  g      @r8  Fr   r2  r3  r9  r  )mambaout_base_tall_rwr   r!  s      r1   r>  r>    sE    !	J i
idS]NhagNhiir2   c                 X    t        ddddddddd	
	      }t        dd| it        |fi |S )Nr)  r-  r7        ?Fr   r2  silur3  	r   r   r{   r|   r   r   r}   r.   r   r  )mambaout_base_wide_rwr   r!  s      r1   rC  rC    H    !
J i
idS]NhagNhiir2   c                 X    t        ddddddddd	
	      }t        dd| it        |fi |S )Nr<  r-  r7  r@  Fr   r2  rA  r3  rB  r  )mambaout_base_plus_rwr   r!  s      r1   rF  rF    rD  r2   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )N)r   r   r   r   )       r  @   r   Fr   g-C6?rA  r3  )r   r   r{   r   r   r}   r.   r   r  )r  r   r!  s      r1   r  r    sD    	J a
ad:F`Y_F`aar2   )r   rl   )4r;   collectionsr   typingr   r   r   r   r   r   	timm.datar
   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   r   r   rA   rL   rP   ro   r   r   r   r
  default_cfgsr  r  r%  r'  r+  r0  r4  r:  r>  rC  rF  r   r2   r1   <module>rU     sx   $ / /   A n n * + ' <*299 *Z")) 4 49bii 9x2RYY 2j0BII 0fSryy Sl2 % '+' ,0S, +/S+ +/S+ 484 9= -8_g9 ,0, ]M]cdQ)& )X c c
 b b b b
 c c
 b b
 	f 	f k k j j j j j j b br2   