
    kh+                        d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlm Z m!Z! dgZ" G d dejF                        Z$ G d dejF                        Z% G d dejF                        Z& G d dejN                        Z( G d dejN                        Z) G d dejF                        Z* G d dejF                        Z+ G d dejF                        Z, G d dejF                        Z- G d  d!ejF                        Z. G d" d#ejF                        Z/ G d$ d%ejF                        Z0 G d& dejF                        Z1 G d' d(e1      Z2d) Z3 e4 e4d*d+d,d-.       e4d*d+d/d0.       e4d1d2d3d0.       e4d4d2d,d0.       e4d5d2d6d0.       e4d5d2d6d0d7d89       e4d:d;d<d0d7d89       e4d=d;d>d0d7?       e4d4d2d,d@d7?       e4dAd;dBd@d7?      C
      Z5dsdDZ6dtdEZ7 e i dF e7dGH      dI e7dGH      dJ e7dGH      dK e7dGH      dL e7dGH      dM e7dGdNO      dP e7dGdNO      dQ e7dGdNO      dR e7dGdNO      dS e7dGdNO      dT e7dUV      dW e7dUV      dX e7dUV      dY e7dUV      dZ e7dUV      d[ e7dUV      d\ e7dUV       e7dUV       e7dUV       e7dUV      d]      Z8e!dud^e1fd_       Z9e!dud^e1fd`       Z:e!dud^e1fda       Z;e!dud^e1fdb       Z<e!dud^e1fdc       Z=e!dud^e1fdd       Z>e!dud^e1fde       Z?e!dud^e1fdf       Z@e!dud^e1fdg       ZAe!dud^e1fdh       ZBe!dud^e1fdi       ZCe!dud^e1fdj       ZDe!dud^e1fdk       ZEe!dud^e1fdl       ZFe!dud^e1fdm       ZGe!dud^e1fdn       ZHe!dud^e1fdo       ZIe!dud^e1fdp       ZJe!dud^e1fdq       ZKe!dud^e1fdr       ZLy)va   LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OrderedDict)partial)DictListOptionalTupleUnionN)IMAGENET_DEFAULT_STDIMAGENET_DEFAULT_MEAN)	to_ntuple	to_2tupleget_act_layerDropPathtrunc_normal_ndgrid   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_modelLevitc                   V     e Zd Z	 d fd	Z ej
                         d        Zd Z xZS )ConvNormc	           
         t         	|           t        j                  |||||||d      | _        t        j
                  |      | _        t        j                  j                  | j                  j                  |       y NFbias)
super__init__nnConv2dlinearBatchNorm2dbninit	constant_weight)
selfin_chsout_chskernel_sizestridepaddingdilationgroupsbn_weight_init	__class__s
            M/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/levit.pyr!   zConvNorm.__init__,   sZ    iifgxY_fkl..)
$''...9    c           	          | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d      |j                  d      |j                  dd  | j                   j                  | j                   j                  | j                   j                  | j                   j                        }|j                  j                  j!                  |       |j
                  j                  j!                  |       |S )N      ?r   r      )r.   r/   r0   r1   )r$   r&   r)   running_varepsr   running_meanr"   r#   sizeshaper.   r/   r0   r1   datacopy_)r*   cr&   wbms         r4   fusezConvNorm.fuse4   s   TWW2II"&&0S88HHqD$,--GGboo		1R^^bff5LQT4TTTIIFF1Iqvvay!''!"+dkk6H6HKK''$++2F2Ft{{OaOac 	
A	!r5   c                 B    | j                  | j                  |            S N)r&   r$   r*   xs     r4   forwardzConvNorm.forwardA   s    wwt{{1~&&r5   )r   r   r   r   r   r   	__name__
__module____qualname__r!   torchno_gradrD   rI   __classcell__r3   s   @r4   r   r   +   s,    lm: U]]_
 
'r5   r   c                   T     e Zd Zd fd	Z ej
                         d        Zd Z xZS )
LinearNormc                     t         |           t        j                  ||d      | _        t        j
                  |      | _        t        j                  j                  | j                  j                  |       y r   )
r    r!   r"   Linearr$   BatchNorm1dr&   r'   r(   r)   )r*   in_featuresout_featuresr2   r3   s       r4   r!   zLinearNorm.__init__F   sN    ii\F...
$''...9r5   c                 6   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d      |j                  d            }|j                  j                  j                  |       |j
                  j                  j                  |       |S )Nr7   r   r   )r$   r&   r)   r9   r:   r   r;   r"   rU   r<   r>   r?   )r*   lr&   rA   rB   rC   s         r4   rD   zLinearNorm.fuseM   s    TWW2II"&&0S88HHqDz!GGboo		1R^^bff5LQT4TTTIIaffQi+	A	!r5   c                     | j                  |      }| j                  |j                  dd            j                  |      S )Nr   r   )r$   r&   flatten
reshape_asrG   s     r4   rI   zLinearNorm.forwardX   s3    KKNwwqyyA'22155r5   )r   rJ   rQ   s   @r4   rS   rS   E   s)    : U]]_ 6r5   rS   c                   T     e Zd Zd fd	Z ej
                         d        Zd Z xZS )
NormLinearc                    t         |           t        j                  |      | _        t        j
                  |      | _        t        j                  |||      | _        t        | j                  j                  |       | j                  j                  5t        j                  j                  | j                  j                  d       y y )Nr   )stdr   )r    r!   r"   rV   r&   DropoutdroprU   r$   r   r)   r   r'   r(   )r*   rW   rX   r   ra   rc   r3   s         r4   r!   zNormLinear.__init__^   s    ..-JJt$	ii\Edkk((c2;;'GGdkk..2 (r5   c                 6   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                   j                  | j                   j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d f   z  }|j
                  $|| j                  j                  j                  z  }n<|j                  |d d d f   z  j                  d      | j                  j
                  z   }t        j                  |j                  d      |j                  d            }|j                  j                  j                  |       |j
                  j                  j                  |       |S )Nr7   r   r   )r&   r$   r)   r9   r:   r   r;   Tviewr"   rU   r<   r>   r?   )r*   r&   rZ   rA   rB   rC   s         r4   rD   zNormLinear.fuseh   s-   AII"&&0S88GGdgg**TWW^^;r~~PRPVPV?V[^>^^^HHqqz!66>DKK&&(((AAagJ&,,R04;;3C3CCAIIaffQi+	A	!r5   c                 `    | j                  | j                  | j                  |                  S rF   )r$   rc   r&   rG   s     r4   rI   zNormLinear.forwardw   s"    {{499TWWQZ011r5   )Tg{Gz?        rJ   rQ   s   @r4   r_   r_   ]   s)    3 U]]_ 2r5   r_   c                        e Zd Z fdZ xZS )Stem8c           
      d   t         |           d| _        | j                  dt	        ||dz  ddd             | j                  d |              | j                  d	t	        |dz  |dz  ddd             | j                  d
 |              | j                  dt	        |dz  |ddd             y )N   conv1      r8   r   r.   r/   act1conv2act2conv3r    r!   r.   
add_moduler   r*   r+   r,   	act_layerr3   s       r4   r!   zStem8.__init__|   s    &'Q,!UV!WX	,'Q,1aPQ[\!]^	,'Q,1VW!XYr5   rK   rL   rM   r!   rP   rQ   s   @r4   rk   rk   {   s    Z Zr5   rk   c                        e Zd Z fdZ xZS )Stem16c           
         t         |           d| _        | j                  dt	        ||dz  ddd             | j                  d |              | j                  d	t	        |dz  |d
z  ddd             | j                  d |              | j                  dt	        |d
z  |dz  ddd             | j                  d |              | j                  dt	        |dz  |ddd             y )N   rn   rm   rp   r8   r   rq   rr   rs   ro   rt   ru   act3conv4rv   rx   s       r4   r!   zStem16.__init__   s    &'Q,!UV!WX	,'Q,1aPQ[\!]^	,'Q,1aPQ[\!]^	,'Q,1VW!XYr5   rz   rQ   s   @r4   r|   r|      s    
Z 
Zr5   r|   c                   &     e Zd Zd fd	Zd Z xZS )
Downsamplec                     t         |           || _        t        |      | _        |rt        j                  d|dd      | _        y d | _        y )Nrp   r   F)r.   r/   count_include_pad)r    r!   r.   r   
resolutionr"   	AvgPool2dpool)r*   r.   r   use_poolr3   s       r4   r!   zDownsample.__init__   sA    #J/ZbBLL61PUV	hl	r5   c                 p   |j                   \  }}}|j                  || j                  d   | j                  d   |      }| j                  6| j                  |j	                  dddd            j	                  dddd      }n$|d d d d | j
                  d d | j
                  f   }|j                  |d|      S )Nr   r   rp   r8   re   )r=   rg   r   r   permuter.   reshape)r*   rH   BNCs        r4   rI   zDownsample.forward   s    ''1aFF1dooa($//!*<a@99 		!))Aq!Q/088Aq!DA!]t{{]MdkkM12AyyB""r5   F)rK   rL   rM   r!   rI   rP   rQ   s   @r4   r   r      s    m#r5   r   c                        e Zd ZU eeej                  f   ed<   ddddej                  f fd	Z
 ej                         d fd	       Zdej                  d	ej                  fd
Zd Z xZS )	Attentionattention_bias_cacherm         @   Fc                    t         |           |rt        nt        }t	        |      }|| _        || _        |dz  | _        || _        ||z  | _	        t        ||z        | _        t        ||z        |z  | _         ||| j                  | j                  dz  z         | _        t        j                  t!        d |       fd || j                  |d      fg            | _        t        j$                  t'        j(                  ||d   |d   z              | _        t'        j,                  t/        t'        j0                  |d         t'        j0                  |d                     j3                  d      }	|	dd d d f   |	dd d d f   z
  j5                         }
|
d   |d   z  |
d   z   }
| j7                  d	|
d
       i | _        y )N      r8   actlnr   r2   r   .attention_bias_idxsF
persistent)r    r!   r   rS   r   use_conv	num_headsscalekey_dimkey_attn_dimintval_dimval_attn_dimqkvr"   
Sequentialr   proj	ParameterrN   zerosattention_biasesstackr   aranger\   absregister_bufferr   )r*   dimr   r   
attn_ratior   r   ry   ln_layerposrel_posr3   s              r4   r!   zAttention.__init__   s    	'8Zz*
 "_
#i/:/0
W 45	AC!2!2T5F5F5J!JKMM+IK 8D--s1EF/
 # 	
 !#U[[JqMT^_`TaDa-b ckk&jm!<ell:VW=>YZ[ccdefsAt|$s3a<'88==?1:
1-;2GN$&!r5   c                 R    t         |   |       |r| j                  ri | _        y y y rF   r    trainr   r*   moder3   s     r4   r   zAttention.train   )    dD--(*D% .4r5   devicereturnc                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rF   rN   jit
is_tracingtrainingr   r   strr   r*   r   
device_keys      r4   get_attention_biaseszAttention.get_attention_biases       99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r5   c                 ^   | j                   r|j                  \  }}}}| j                  |      j                  || j                  d||z        j                  | j                  | j                  | j                  gd      \  }}}|j                  dd      |z  | j                  z  | j                  |j                        z   }	|	j                  d      }	||	j                  dd      z  j                  |d||      }n|j                  \  }}
}| j                  |      j                  ||
| j                  d      j                  | j                  | j                  | j                  gd      \  }}}|j                  dddd      }|j                  dddd      }|j                  dddd      }||z  | j                  z  | j                  |j                        z   }	|	j                  d      }	|	|z  j                  dd      j                  ||
| j                        }| j!                  |      }|S )Nre   r8   r   rp   r   r   )r   r=   r   rg   r   splitr   r   	transposer   r   r   softmaxr   r   r   r   )r*   rH   r   r   HWqkvattnr   s              r4   rI   zAttention.forward   s   ==JAq!Qhhqk&&4>>2q1u..3eT\\4<<QUQ]Q]4^dee.f Aq! KKB'!+tzz9D<U<UVWV^V^<__D<<B<'DT^^B++11!RA>AggGAq!hhqk&&1dnnb**/%t||T\\0Z`a%*b Aq!		!Q1%A		!Q1%A		!Q1%Aq54::%(A(A!(((KKD<<B<'D$$Q*221a9J9JKAIIaLr5   TrK   rL   rM   r   r   rN   Tensor__annotations__r"   SiLUr!   rO   r   r   r   rI   rP   rQ   s   @r4   r   r      sn    sELL011 gg!'F U]]_+ +
95<< 9ELL 9r5   r   c                        e Zd ZU eeej                  f   ed<   ddddddej                  f fd	Z
 ej                         d fd	       Zd	ej                  d
ej                  fdZd Z xZS )AttentionDownsampler   rm          @r8   r   Fc           
      r   t         |           t        |      }|| _        || _        || _        || _        ||z  | _        t        ||z        | _	        | j                  | j
                  z  | _
        |dz  | _        || _        | j                  r,t        }t        t        j                   |	rdnd|	rdndd      }nt"        }t        t$        ||	      } ||| j                  | j                  z         | _        t        j(                  t+        d ||	      fd
 ||| j                        fg            | _        t        j(                  t+        d |
       fd
 || j                  |      fg            | _        t        j0                  t3        j4                  ||d   |d   z              | _        t3        j8                  t;        t3        j<                  |d         t3        j<                  |d                     j?                  d      }t3        j8                  t;        t3        j<                  d|d   |      t3        j<                  d|d   |                  j?                  d      }|dd d d f   |dd d d f   z
  jA                         }|d   |d   z  |d   z   }| jC                  d|d       i | _"        y )Nr   rp   r   r   F)r-   r/   r   )r   r   down)r.   r   r   )step.r   r   )#r    r!   r   r.   r   r   r   r   r   r   r   r   r   r   r   r"   r   rS   r   kvr   r   r   r   r   rN   r   r   r   r   r   r\   r   r   r   )r*   in_dimout_dimr   r   r   r.   r   r   r   ry   r   	sub_layerk_posq_posr   r3   s                   r4   r!   zAttentionDownsample.__init__   sq    	z*
$"#i/:/0 LL4>>9_
 ==H!)Aqx!QbgiI "H
zHUI64#4#4t7H7H#HI{Yf-.8FD$5$567,
    MM+IK 8D--w78/
 # 	
 !#U[[JqMT^_`TaDa-b cF5<<
1#>ZXY]@[\]eefghFLLJqM7LLJqM7
  71: 	 a&sD!|)<<AAC1:
1-;2GN$&!r5   c                 R    t         |   |       |r| j                  ri | _        y y y rF   r   r   s     r4   r   zAttentionDownsample.train1  r   r5   r   r   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rF   r   r   s      r4   r   z(AttentionDownsample.get_attention_biases7  r   r5   c                 p   | j                   rO|j                  \  }}}}|dz
  | j                  z  dz   |dz
  | j                  z  dz   }}| j                  |      j	                  || j
                  d||z        j                  | j                  | j                  gd      \  }}	| j                  |      j	                  || j
                  | j                  d      }
|
j                  dd      |z  | j                  z  | j                  |j                        z   }|j                  d      }|	|j                  dd      z  j                  || j                   ||      }nH|j                  \  }}}| j                  |      j	                  ||| j
                  d      j                  | j                  | j                  gd      \  }}	|j#                  dddd      }|	j#                  dddd      }	| j                  |      j	                  |d| j
                  | j                        j#                  dddd      }
|
|z  | j                  z  | j                  |j                        z   }|j                  d      }||	z  j                  dd      j                  |d| j                         }| j%                  |      }|S )Nr   re   r8   r   r   rp   r   )r   r=   r.   r   rg   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r*   rH   r   r   r   r   HHWWr   r   r   r   r   s                r4   rI   zAttentionDownsample.forward@  sK   ==JAq!Q!e+a/!a%DKK1G!1KB771:??1dnnb!a%@FFVZVbVbGcijFkDAqq	q$..$,,CAKKB'!+tzz9D<U<UVWV^V^<__D<<B<'DT^^B++44Q8I8I2rRAggGAq!771:??1a<BBDLLRVR^R^C_efBgDAq		!Q1%A		!Q1%Aq	q"dnndllCKKAqRSUVWAq54::%(A(A!(((KKD<<B<'D$$Q*221b$:K:KLAIIaLr5   r   r   rQ   s   @r4   r   r      st    sELL011 gg7'r U]]_+ +
95<< 9ELL 9r5   r   c                   H     e Zd ZdZdddej
                  df fd	Zd Z xZS )LevitMlpzL MLP for Levit w/ normalization + ability to switch btw conv and linear
    NFri   c                     t         |           |xs |}|xs |}|rt        nt        } |||      | _         |       | _        t        j                  |      | _         |||d      | _	        y )Nr   r   )
r    r!   r   rS   ln1r   r"   rb   rc   ln2)	r*   rW   hidden_featuresrX   r   ry   rc   r   r3   s	           r4   r!   zLevitMlp.__init__]  sd     	#2{)8['8ZK9;JJt$	O\!Lr5   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rF   )r   r   rc   r   rG   s     r4   rI   zLevitMlp.forwardp  s;    HHQKHHQKIIaLHHQKr5   )	rK   rL   rM   __doc__r"   r   r!   rI   rP   rQ   s   @r4   r   r   Z  s(    
 !ggM&r5   r   c            	       L     e Zd Zdddej                  dddddf	 fd	Zd	 Z xZS )
LevitDownsamplerm   r   r   Nr   Fri   c                     t         |           |xs |}t        |||||||	|
|	      | _        t	        |t        ||z        |
|      | _        |dkD  rt        |      | _
        y t        j                         | _
        y )N)	r   r   r   r   r   ry   r   r   r   r   ry   ri   )r    r!   r   attn_downsampler   r   mlpr   r"   Identity	drop_path)r*   r   r   r   r   r   	mlp_ratiory   attn_act_layerr   r   r   r   r3   s                r4   r!   zLevitDownsample.__init__y  s     	'492!$!
 
 )#$	
 1:B),BKKMr5   c                 n    | j                  |      }|| j                  | j                  |            z   }|S rF   )r   r   r   rG   s     r4   rI   zLevitDownsample.forward  s2      #txx{++r5   rK   rL   rM   r"   r   r!   rI   rP   rQ   s   @r4   r   r   x  s0     gg$RLr5   r   c                   J     e Zd Zdddddej                  ddf fd	Zd	 Z xZS )

LevitBlockrm   r   r   r   FNri   c           	      L   t         |           |	xs |}	t        |||||||	      | _        |
dkD  rt	        |
      nt        j                         | _        t        |t        ||z        ||      | _
        |
dkD  rt	        |
      | _        y t        j                         | _        y )N)r   r   r   r   r   r   ry   ri   r   )r    r!   r   r   r   r"   r   
drop_path1r   r   r   
drop_path2)r*   r   r   r   r   r   r   r   ry   r   r   r3   s              r4   r!   zLevitBlock.__init__  s     	'49!!$	 2;R(9-R[[]i 	
 2;R(9-R[[]r5   c                     || j                  | j                  |            z   }|| j                  | j                  |            z   }|S rF   )r   r   r   r   rG   s     r4   rI   zLevitBlock.forward  s=    		!--,,r5   r   rQ   s   @r4   r   r     s-    
 gg!SFr5   r   c            
       N     e Zd Zddddej                  dddddf
 fd		Zd
 Z xZS )
LevitStagero   rm   r   Nr    Fri   c                 x   t         |           t        |
      }
|r7t        |||||z  dd||	|
||      | _        |
D cg c]  }|dz
  dz  dz    }
}n ||k(  sJ t        j                         | _        g }t        |      D ]  }|t        |||||||	|
||
      gz  } t        j                  | | _
        y c c}w )Nr   r   )	r   r   r   r   ry   r   r   r   r   r   r8   )r   r   r   ry   r   r   r   r   )r    r!   r   r   
downsampler"   r   ranger   r   blocks)r*   r   r   r   depthr   r   r   ry   r   r   r  r   r   rr  _r3   s                    r4   r!   zLevitStage.__init__  s      	z*
- G+#-%!#DO 5??q1q5Q,*?J?W$$$ kkmDOu 	Az#%##-%!#  F	 mmV,' @s   B7c                 J    | j                  |      }| j                  |      }|S rF   )r  r  rG   s     r4   rI   zLevitStage.forward  s"    OOAKKNr5   r   rQ   s   @r4   r  r    s2     gg4-lr5   r  c                   \    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zej                  j                  d        Zej                  j                  dd       Z	ej                  j                  dd       Z
ej                  j                  dej                  fd       Zdded	ee   fd
Z	 	 	 	 	 ddej$                  deeeee   f      dededededeeej$                     eej$                  eej$                     f   f   fdZ	 	 	 ddeeee   f   dedefdZd ZddefdZd Z xZS )r   z Vision Transformer with support for patch or hybrid CNN input stage

    NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
    w/ train scripts that don't take tuple outputs,
    c                    t         |           t        |      }t        |xs |      }|| _        || _        || _        |d   x| _        | _        || _        || _	        d| _
        g | _        t        |      }t        |      |k(  sJ  t        |      |      } t        |      |      } t        |      |	      }	|
|dk\  sJ |
| _        |}nN|dv sJ |dk(  rt        ||d   |      | _        nt!        ||d   |      | _        | j                  j"                  }t%        t'        t)        |      t)        |            D cg c]
  \  }}||z   c}}      }|d   }g }t+        |      D ]  }|dkD  rdnd}|t-        |||   |||   ||   ||   |	|   |||||dk(  r|nd	|
      gz  }||z  }t%        |D cg c]  }|dz
  |z  dz    c}      }| xj                  t/        ||   |d|       gz  c_        ||   } t1        j2                  | | _        |dkD  rt7        |d   ||      | _        y t1        j8                         | _        y c c}}w c c}w )Nre   Fr8   )s16s8r  r   )ry   r   r  )
r	  r   r   r   ry   r   r   r   r  r   zstages.)num_chs	reductionmodulerc   )r    r!   r   r   num_classesglobal_poolnum_featureshead_hidden_size	embed_dim	drop_rategrad_checkpointingfeature_infolenr   stemr|   rk   r.   tuplezipr   r  r  dictr"   r   stagesr_   r   head)r*   img_sizein_chansr  r  r   r	  r   r   r   stem_backbonestem_stride	stem_typedown_opry   r   r   r  r  drop_path_rate
num_stagesr.   ipr   r   r"  stage_strider
  r3   s                                r4   r!   zLevit.__init__  s   * 	!),	&~'BC &&4=bMAD1"""'^
5zZ''')Ij))4	*Yz*:6
)Ij))4	$!###%DI F---E!"8Yq\YO	!(IaLIN	YY%%Fs9X3F	RXHY/Z[tq!AF[\
1z" 	"A !A11Lz!Ah#A,%a=#A,#-%!&2a&77R(  F l"FZPQ< 7! ;PQJ$y|vX_`a_bVc"d!eeq\F)	"* mmV, OZ\]oJy}k	J	cecncncp	9 \,  Qs   I
6I
c                 n    | j                         j                         D ch c]	  }d|v s| c}S c c}w )Nr   )
state_dictkeysrG   s     r4   no_weight_decayzLevit.no_weight_decayc  s.    ??,113Oa7IQ7NOOOs   	22c                 $    t        dddg      }|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )r  r  )r!  )r*   coarsematchers      r4   group_matcherzLevit.group_matcherg  s    4-/CD
 r5   c                     || _         y rF   )r  r*   enables     r4   set_grad_checkpointingzLevit.set_grad_checkpointingo  
    "(r5   r   c                     | j                   S rF   )r#  r*   s    r4   get_classifierzLevit.get_classifiers  s    yyr5   r  r  c                     || _         ||| _        |dkD  r(t        | j                  || j                        | _        y t        j                         | _        y Nr   r  )r  r  r_   r  r  r"   r   r#  r*   r  r  s      r4   reset_classifierzLevit.reset_classifierw  sS    &"*DDORSO {A	Y[YdYdYf 		r5   rH   indicesnorm
stop_early
output_fmtintermediates_onlyc           	      
   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }|j                  \  }
}}}| j
                  s!|j                  d      j                  dd      }t        j                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]  \  }}| j                  r+t        j                  j                         st        ||      }n ||      }||v rS| j
                  r|j                  |       n5|j                  |j                  |
||d      j!                  dddd             |dz   dz
  dz  }|dz   dz
  dz  } |r|S ||fS )	a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r8   r   Nre   r   rp   )r   r  r"  r  r=   r   r\   r   rN   r   is_scripting	enumerater  r   appendr   r   )r*   rH   rC  rD  rE  rF  rG  intermediatestake_indices	max_indexr   r   r   r   r"  feat_idxstages                    r4   forward_intermediateszLevit.forward_intermediates~  sr   * Y&D(DD&"6s4;;7G"Qi IIaLWW
1a}}		!&&q!,A99!!#:[[F[[)a-0F(0 	!OHe&&uyy/E/E/Gua(!H<'==!((+!((1aB)?)G)G1aQR)STQq AQq A	!   -r5   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r  )r   r  r"  rB  )r*   rC  rS  rT  rN  rO  s         r4   prune_intermediate_layerszLevit.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r5   c                 (   | j                  |      }| j                  s!|j                  d      j                  dd      }| j                  r6t
        j                  j                         st        | j                  |      }|S | j                  |      }|S )Nr8   r   )
r  r   r\   r   r  rN   r   rJ  r   r"  rG   s     r4   forward_featureszLevit.forward_features  ss    IIaL}}		!&&q!,A""599+A+A+Ct{{A.A  AAr5   
pre_logitsc                     | j                   dk(  r0| j                  r|j                  d      n|j                  d      }|r|S | j                  |      S )Navgr   re   r   r   )r  r   meanr#  )r*   rH   rY  s      r4   forward_headzLevit.forward_head  sG    u$(,8$166a6=Aq0DIIaL0r5   c                 J    | j                  |      }| j                  |      }|S rF   )rX  r^  rG   s     r4   rI   zLevit.forward  s'    !!!$a r5   )   rp     )   @   )   )rp   r   r   NNr  	subsample
hard_swishNFr[  ri   ri   r   r   rF   )NFFrI  F)r   FT)rK   rL   rM   r   r!   rN   r   ignorer2  r6  r:  r"   Moduler>  r   r   r   rB  r   r	   r   boolr   rR  rV  rX  r^  rI   rP   rQ   s   @r4   r   r     s    ")Nq` YYP P YY  YY) ) YY		  gC gx} g 8<$$',3 ||3  eCcN343  	3 
 3  3  !%3  
tELL!5tELL7I)I#JJ	K3 n ./$#	3S	>*  	1$ 1
r5   c                        e Zd Z fdZej
                  j                  dej                  fd       Z	d
de
dee   fdZej
                  j                  dd       Zddefd	Z xZS )LevitDistilledc                     t        |   |i | | j                  dkD  r t        | j                  | j                        nt        j                         | _        d| _        y )Nr   F)	r    r!   r  r_   r  r"   r   	head_distdistilled_training)r*   argskwargsr3   s      r4   r!   zLevitDistilled.__init__  sQ    $)&)LPL\L\_`L`D$5$5t7G7GHfhfqfqfs"'r5   r   c                 2    | j                   | j                  fS rF   )r#  rm  r=  s    r4   r>  zLevitDistilled.get_classifier  s    yy$..((r5   r  r  c                    || _         ||| _        |dkD  r"t        | j                  || j                        nt        j                         | _        |dkD  rt        | j                  |      | _        y t        j                         | _        y r@  )	r  r  r_   r  r  r"   r   r#  rm  rA  s      r4   rB  zLevitDistilled.reset_classifier  sz    &"*DDORSO {AY[YdYdYf 		GRUVD$5$5{C\^\g\g\ir5   c                     || _         y rF   )rn  r8  s     r4   set_distilled_trainingz%LevitDistilled.set_distilled_training  r;  r5   rY  c                 P   | j                   dk(  r0| j                  r|j                  d      n|j                  d      }|r|S | j                  |      | j	                  |      }}| j
                  r.| j                  r"t        j                  j                         s||fS ||z   dz  S )Nr[  r\  r   r   r8   )
r  r   r]  r#  rm  rn  r   rN   r   rJ  )r*   rH   rY  x_dists       r4   r^  zLevitDistilled.forward_head  s    u$(,8$166a6=AHIIaL$.."36""t}}UYY=S=S=Uf9 J!##r5   rF   r   r   )rK   rL   rM   r!   rN   r   rg  r"   rh  r>  r   r   r   rB  rt  ri  r^  rP   rQ   s   @r4   rk  rk    su    (
 YY)		 ) )jC jhsm j YY) )$$ $r5   rk  c                 t    d| v r| d   } | j                         D ci c]  \  }}d|vs|| } }}| S c c}}w )Nmodelr   )items)r0  rx  r   r   s       r4   checkpoint_filter_fnrz    sQ    *(
 $.#3#3#5X41a9NVW9W!Q$XJX  Ys   44)        r~   )ro      rm   )r8   rp   ro   )r  r   r   r	  )ro   rm   rd  )ro   ro   ro   )rb  i   r}      )rp      r~  )r|  r}     )r}  r     )r~  	   rd  silur  )r  r   r   r	  ry   r(  )r    i  rc  )rm   
   r   )r  r  i   )rm   rd  r~   )r  r   r   r	  ry   )ro   rm   r~  )r  r  r  )rm   r  rd  )

levit_128s	levit_128	levit_192	levit_256	levit_384levit_384_s8levit_512_s8	levit_512
levit_256d
levit_512dc                 >   d| v }|j                  dd      }|j                  dd      r|s|j                  dd       || t        v r| }n|r| j	                  dd      }t        t        |   fi |}t        |rt        nt        | |ft        t        d	|
      d|}|S )N_convout_indices)r   r   r8   features_onlyFfeature_clsgetterr  T)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)
popget
setdefault
model_cfgsreplacer!  r   rk  r   rz  )	variantcfg_variant
pretrained	distilledrp  is_convr  	model_cfgrx  s	            r4   create_levitr  /  s     G**]I6Kzz/5)'-2j !K!//'26KZ,77I # 2DkJ E Lr5   c                 2    | ddd dddt         t        ddd|S )	Nra  )rp   r`  r`  g?bicubicTzstem.conv1.linear)head.linearzhead_dist.linear)urlr  
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer]  ra   
first_conv
classifier)r   r
   )r  rp  s     r4   _cfgr  F  s3    =t%.B)9\  r5   zlevit_128s.fb_dist_in1kztimm/)	hf_hub_idzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1k)ro   ro   )r  r  zlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedr  )r  zlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrained)zlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrainedr   c                     t        dd| i|S )Nr  )r  r  r  rp  s     r4   r  r    s    FFvFFr5   c                     t        dd| i|S )Nr  )r  r  r  s     r4   r  r        E
EfEEr5   c                     t        dd| i|S )Nr  )r  r  r  s     r4   r  r    r  r5   c                     t        dd| i|S )Nr  )r  r  r  s     r4   r  r    r  r5   c                     t        dd| i|S )Nr  )r  r  r  s     r4   r  r    r  r5   c                     t        dd| i|S )Nr  )r  r  r  s     r4   r  r    s    H:HHHr5   c                     t        d| dd|S )NFr  r  )r  r  r  s     r4   r  r    s    Y:YRXYYr5   c                     t        d| dd|S )NFr  )r  r  r  s     r4   r  r    s    V
eVvVVr5   c                     t        d| dd|S )NFr  )r  r  r  s     r4   r  r        WuWPVWWr5   c                     t        d| dd|S )NFr  )r  r  r  s     r4   r  r    r  r5   c                     t        d| dd|S )NTr  r   )levit_conv_128sr  r  s     r4   r  r    s    Zj4ZSYZZr5   c                     t        d| dd|S )NTr  )levit_conv_128r  r  s     r4   r  r        YZ$YRXYYr5   c                     t        d| dd|S )NTr  )levit_conv_192r  r  s     r4   r  r    r  r5   c                     t        d| dd|S )NTr  )levit_conv_256r  r  s     r4   r  r    r  r5   c                     t        d| dd|S )NTr  )levit_conv_384r  r  s     r4   r  r    r  r5   c                     t        d| dd|S )NTr  )levit_conv_384_s8r  r  s     r4   r  r    s    \
T\U[\\r5   c                      t        d| ddd|S )NTFr  r   r  )levit_conv_512_s8r  r  s     r4   r  r    s    m
T]bmflmmr5   c                      t        d| ddd|S )NTFr  )levit_conv_512r  r  s     r4   r  r    s    jZ$Z_jcijjr5   c                      t        d| ddd|S )NTFr  )levit_conv_256dr  r  s     r4   r  r        kj4[`kdjkkr5   c                      t        d| ddd|S )NTFr  )levit_conv_512dr  r  s     r4   r  r    r  r5   )NFT)r  r   )Mr   collectionsr   	functoolsr   typingr   r   r   r   r	   rN   torch.nnr"   	timm.datar
   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__rh  r   rS   r_   r   rk   r|   r   r   r   r   r   r   r  r   rk  rz  r!  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r5   r4   <module>r     s  2 $  5 5   A \ \ * + 3 <)'ryy '46 602 2<	ZBMM 	ZZR]] Z# #"L		 L^b")) bJryy <*bii *Z' 'T: :zGBII GT!$U !$H* !2)U!29V!2)U!2)U!29V !29D* !2ID*
 "B+Ybhj !2)_eg!2Iagi7
@. % 3&t 3&
 d3& d3& d3& d3&& #D%'3&. "4$/3&6 "4$73&> "4$?3&F "4$G3&P dm<Q3&R dm<S3&T 4=9U3&V DM:W3&X DM:Y3&\ "4=#A]3&^ "4=#A_3&` !% >!%!?!%!?e3& 3l Ge G G FU F F FU F F FU F F FU F F I I I Z Z Z WU W W Xe X X Xe X X [5 [ [ Z% Z Z Z% Z Z Z% Z Z Z% Z Z ]U ] ] nU n n k% k k l5 l l l5 l lr5   