
    khf                        d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z
 ddlmZmZmZmZmZmZmZmZmZmZmZ g dZeej0                     Zdee   defd	Z G d
 dej0                        Z G d dej0                        Z G d dej0                        Z G d dej0                        Z G d dej0                        Z  G d dej0                        Z! G d dej0                        Z" G d de      Z# G d dej0                        Z$y)z[ EfficientNet, MobileNetV3, etc Blocks

Hacked together by / Copyright 2019, Ross Wightman
    )CallableDictOptionalTypeN)
functional)create_conv2dDropPathmake_divisiblecreate_act_layer	create_aa	to_2tuple	LayerTypeConvNormActget_norm_act_layerMultiQueryAttention2dAttention2d)SqueezeExcite	ConvBnActDepthwiseSeparableConvInvertedResidualCondConvResidualEdgeResidualUniversalInvertedResidualMobileAttention
group_sizechannelsc                 &    | sy|| z  dk(  sJ || z  S )N   r    )r   r   s     \/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/_efficientnet_blocks.py
num_groupsr!      s(     *$))):%%    c                        e Zd ZdZddej
                  ej                  ddfdedede	e   de
de
d	e	e
   d
e	e   f fdZd Z xZS )r   a   Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family

    Args:
        in_chs (int): input channels to layer
        rd_ratio (float): ratio of squeeze reduction
        act_layer (nn.Module): activation layer of containing block
        gate_layer (Callable): attention gate function
        force_act_layer (nn.Module): override block's activation fn if this is set/bound
        rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs
    g      ?Nin_chsrd_ratiord_channels	act_layer
gate_layerforce_act_layerrd_round_fnc                     t         t        |           ||xs t        } |||z        }|xs |}t	        j
                  ||dd      | _        t        |d      | _        t	        j
                  ||dd      | _	        t        |      | _
        y )Nr   T)biasinplace)superr   __init__roundnnConv2dconv_reducer   act1conv_expandgate)	selfr$   r%   r&   r'   r(   r)   r*   	__class__s	           r    r0   zSqueezeExcite.__init__+   s     	mT+-%.K%fx&78K#0y	99V[!$G$Y=	99[&!$G$Z0	r"   c                     |j                  dd      }| j                  |      }| j                  |      }| j                  |      }|| j	                  |      z  S )N)      T)keepdim)meanr4   r5   r6   r7   )r8   xx_ses      r    forwardzSqueezeExcite.forward?   sT    vvfdv+%yy%499T?""r"   )__name__
__module____qualname____doc__r2   ReLUSigmoidintfloatr   r   r   r0   rA   __classcell__r9   s   @r    r   r      s    	 #)-#%77$&JJ37.211 1 "#	1
 !1 "1 &i01 "(+1(#r"   r   c                        e Zd ZdZdddddej
                  ej                  ddf	ded	ed
ededededede	de
de
dee
   def fdZd Zd Z xZS )r   z@ Conv + Norm Layer + Activation w/ optional skip connection
    r   r    FN        r$   out_chskernel_sizestridedilationr   pad_typeskipr'   
norm_layeraa_layerdrop_path_ratec           	      n   t         t        |           t        |
|	      }t	        ||      }|xr |dk(  xr ||k(  | _        |d uxr |dkD  }t        ||||rdn||||      | _         ||d      | _        t        ||||      | _
        |rt        |      | _        y t        j                         | _        y )Nr   rQ   rR   groupspaddingTr-   r   rQ   enable)r/   r   r0   r   r!   has_skipr   convbn1r   aar	   r2   Identity	drop_path)r8   r$   rO   rP   rQ   rR   r   rS   rT   r'   rU   rV   rW   norm_act_layerrZ   use_aar9   s                   r    r0   zConvBnAct.__init__J   s     	i')+J	BJ/B1B71B%4&1*!G[1Ffh@	 "'48HwvfU5C.1r"   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S )N	expansionr`   rA   module	hook_typenum_chsrM   ri   rk   )dictr_   out_channelsr8   locations     r    feature_infozConvBnAct.feature_infog   s:    {"u	499CYCYZZr499+A+ABBr"   c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  r| j	                  |      |z   }|S N)r_   r`   ra   r^   rc   r8   r?   shortcuts      r    rA   zConvBnAct.forwardm   sM    IIaLHHQKGGAJ==q!H,Ar"   rB   rC   rD   rE   r2   rF   BatchNorm2drH   strboolr   r   rI   r0   rq   rA   rJ   rK   s   @r    r   r   G   s     #%77$&NN,0$&WW W 	W
 W W W W W !W "W y)W "W:Cr"   r   c            !            e Zd ZdZdddddddddej
                  ej                  dddfd	ed
edededededede	dede	dede
de
dee
   dee   def  fdZd Zd Z xZS )r   z Depthwise-separable block
    Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion
    (factor of 1.0). This is an alternative to having a IR with an optional first pw conv.
    r<   r   rM   Fr   NrN   r$   rO   dw_kernel_sizerQ   rR   r   rS   noskippw_kernel_sizepw_acts2dr'   rU   rV   se_layerrW   c           	         t         t        |           t        ||      }|dk(  xr ||k(  xr | | _        |
| _        |d uxr |dkD  }|dk(  rGt        |dz        }t        ||ddd      | _         |||      | _	        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _	        |}t        ||      }t        ||||rdn||||      | _         ||d	      | _        t        ||||
      | _        |r
 |||      nt        j                          | _        t        |||	|      | _         ||d| j
                        | _        |rt)        |      | _        y t        j                          | _        y )Nr      r;   samerP   rQ   r[   F)rQ   rR   r[   rZ   Tr-   r\   r'   r[   )r.   	apply_act)r/   r   r0   r   r^   
has_pw_actrH   r   conv_s2dbn_s2dr!   conv_dwr`   r   ra   r2   rb   seconv_pwbn2r	   rc   )r8   r$   rO   r{   rQ   rR   r   rS   r|   r}   r~   r   r'   rU   rV   r   rW   rd   re   sd_chsdw_pad_typerZ   r9   s                         r    r0   zDepthwiseSeparableConv.__init__|   st   & 	$d46+J	B1:7):JF
 %4&1* !8!_F)&&aPQ[abDM(8DK,q0Q6N$2a$7&XKFF DMDK"KJ/$FN1F{6C "&$7HwvfU <D(6Y7$VWnhW!'44??S5C.1r"   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S Nrg   r   forward_prerh   rM   rl   rm   r   in_channelsrn   ro   s     r    rq   z#DepthwiseSeparableConv.feature_info   :    {"yM4<<KcKcddr4<<+D+DEEr"   c                 r   |}| j                   "| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rs   )
r   r   r   r`   ra   r   r   r   r^   rc   rt   s      r    rA   zDepthwiseSeparableConv.forward   s    ==$a AAALLOHHQKGGAJGGAJLLOHHQK==q!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   r   r   
ModuleTyperI   r0   rq   rA   rJ   rK   s   @r    r   r   w   s    #$ "# #%77$&NN,0-1$&#5W5W 5W  	5W
 5W 5W 5W 5W 5W  5W 5W 5W !5W "5W y)5W  z*!5W" "#5WnFr"   r   c            %            e Zd ZdZddddddddddej
                  ej                  dddd	fd
ededededededede	de
dedededededee   dee   dee   de
f$ fdZd Zd Z xZS )r   a   Inverted residual block w/ optional SE

    Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often
    referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in
      * MNasNet - https://arxiv.org/abs/1807.11626
      * EfficientNet - https://arxiv.org/abs/1905.11946
      * MobileNet-V3 - https://arxiv.org/abs/1905.02244
    r<   r   rM   F      ?r   NrN   r$   rO   r{   rQ   rR   r   rS   r|   	exp_ratioexp_kernel_sizer}   r   r'   rU   rV   r   conv_kwargsrW   c                 ,   t         t        |           t        ||      }|xs i }||k(  xr |dk(  xr | | _        |d uxr |dkD  }|dk(  rGt        |dz        }t        ||ddd      | _         |||      | _        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _        |}t        ||	z        }t        ||      }t        |||
fd|i|| _         ||d	      | _        t        |||f|rdn||||d
|| _         ||d	      | _        t        ||||      | _        |r
 |||      nt#        j$                         | _        t        |||fd|i|| _         ||d      | _        |rt-        |      | _        y t#        j$                         | _        y )Nr   r   r;   r   r   Fr[   Tr-   rY   r\   r   r   )r/   r   r0   r   r^   rH   r   r   r   r
   r!   r   r`   r   r   r   ra   r2   rb   r   conv_pwlbn3r	   rc   )r8   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r   r'   rU   rV   r   r   rW   rd   re   r   r   mid_chsrZ   r9   s                            r    r0   zInvertedResidual.__init__   s   * 	.0+J	B!'R7*:v{JF
%4&1* !8!_F)&&aPQ[abDM(8DK,q0Q6N$2a$7&XKFF DMDK"K )!34J0 %VWogxg[fg!'48 %WnR1FfkR FQR "'48HwvfU =E(7i8"++- &gwhPXh\gh!'U;5C.1r"   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S Nrg   r   r   rh   rM   rl   rm   r   r   rn   ro   s     r    rq   zInvertedResidual.feature_info  :    {"z]DMMLeLeffr4==+E+EFFr"   c                    |}| j                   "| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rs   )r   r   r   r`   r   r   ra   r   r   r   r^   rc   rt   s      r    rA   zInvertedResidual.forward  s    ==$a AAALLOHHQKLLOHHQKGGAJGGAJMM!HHQK==q!H,Ar"   rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r   r0   rq   rA   rJ   rK   s   @r    r   r      s&    #$ "#$"##%77$&NN,0-1*.$&'>W>W >W  	>W
 >W >W >W >W >W >W !>W  >W >W !>W ">W  y)!>W" z*#>W$ "$%>W& "'>W@Gr"   r   c                   4     e Zd Zddededef fdZd Z xZS )LayerScale2ddiminit_valuesr.   c                     t         |           || _        t        j                  |t        j                  |      z        | _        y rs   )r/   r0   r.   r2   	Parametertorchonesgamma)r8   r   r   r.   r9   s       r    r0   zLayerScale2d.__init__,  s2    \\+

3"?@
r"   c                     | j                   j                  dddd      }| j                  r|j                  |      S ||z  S )Nr   )r   viewr.   mul_)r8   r?   r   s      r    rA   zLayerScale2d.forward1  s7    

2q!, $qvve};!e);r"   )h㈵>F)	rB   rC   rD   rH   rI   ry   r0   rA   rJ   rK   s   @r    r   r   +  s&    AC Ae AT A
<r"   r   c            &            e Zd ZdZdddddddddej
                  ej                  dddd	d
fdededededededededede	de
dededee   dee   dee   de
dee
   f$ fdZd Zd Z xZS ) r   a   Universal Inverted Residual Block (aka Universal Inverted Bottleneck, UIB)

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L778
    r   r<   r   rM   Fr   NrN   r   r$   rO   dw_kernel_size_startdw_kernel_size_middw_kernel_size_endrQ   rR   r   rS   r|   r   r'   rU   rV   r   r   rW   layer_scale_init_valuec                    t         t        |           |xs i }||k(  xr |dk(  xr |
 | _        |dkD  r|s|s|sJ |r/|s|nd}t	        ||      }t        |||f||||	d|||d|| _        nt        j                         | _        t        ||z        }t        ||df|	||d|| _
        |r(t	        ||      }t        |||f||||	|||d|| _        nt        j                         | _        |r
 |||      nt        j                         | _        t        ||df|	d||d|| _        |r9|s|s|nd}t	        ||      }|dkD  r|rJ t        |||f||||	d||d|| _        nt        j                         | _        |t        ||      | _        nt        j                         | _        |rt#        |      | _        y t        j                         | _        y )	Nr   F)rQ   rR   rZ   r[   r   r'   rU   rV   )r[   r'   rU   )rQ   rR   rZ   r[   r'   rU   rV   r   )r[   r   r'   rU   )rQ   rR   rZ   r[   r   r'   rU   )r/   r   r0   r^   r!   r   dw_startr2   rb   r
   pw_expdw_midr   pw_projdw_endr   layer_scaler	   rc   )r8   r$   rO   r   r   r   rQ   rR   r   rS   r|   r   r'   rU   rV   r   r   rW   r   dw_start_stridedw_start_groupsr   rZ   dw_end_stridedw_end_groupsr9   s                            r    r0   z"UniversalInvertedResidual.__init__=  sE   * 	'79!'R7*:v{JF
A:'+=ASSS  ,>fAO(V<O' 4&!& #%! DM KKMDM !)!34!GQ
!	

 
 
G4F%"4
! #%!
 
DK ++-DK =E(7i8"++- #Wa
!
 
 *>GYF_`M&z7;Mq ##|%"4
$!$ #%
 
DK ++-DK!-+G5KLD!{{}D5C.1r"   c                     |dk(  r,t        dd| j                  j                  j                        S t        d| j                  j                  j                        S )Nrg   zpw_proj.convr   rh   rM   rl   )rm   r   r_   r   rn   ro   s     r    rq   z&UniversalInvertedResidual.feature_info  sI    {"~PTP\P\PaPaPmPmnnr4<<+<+<+I+IJJr"   c                 8   |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rs   )	r   r   r   r   r   r   r   r^   rc   rt   s      r    rA   z!UniversalInvertedResidual.forward  s    MM!KKNKKNGGAJLLOKKNQ==q!H,Ar"   r   rK   s   @r    r   r   6  s=    )*&'&' "#%77$&NN,0-1*.$&6:'nWnW nW #&	nW
 !$nW !$nW nW nW nW nW nW nW !nW "nW y)nW  z*!nW" "$#nW$ "%nW& %-UO'nW`Kr"   r   c            1            e Zd ZdZdddddddddddddej
                  ej                  d	d
d
d
dddfdedededededededededede	dededede	de
de
dee
   deded ed!ee   d"e	d#e	f0 fd$Zd% Zd& Z xZS )'r   z Mobile Attention Block

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L1504
    r   r<   rM      @   F)r   r   NrN   r   r$   rO   rQ   r{   rR   r   rS   	num_headskey_dim	value_dimuse_multi_queryquery_strides	kv_stridecpe_dw_kernel_sizer|   r'   rU   rV   rW   	attn_drop	proj_dropr   use_biasuse_cpec                    t         t        |           t        ||      }|dk(  xr ||k(  xr | | _        t        |      | _        || _        t        | j                  D cg c]  }|dkD  	 c}      | _	        |rt        ||||dd      | _        nd | _         ||d      | _        |||	z  dk(  sJ ||	z  }|rt        ||||	|
||||||||      | _        nt        ||||||      | _        |t!        ||      | _        nt%        j&                         | _        |rt)        |      | _        y t%        j&                         | _        y c c}w )	Nr   T)rP   rR   	depthwiser,   Fr   r   )dim_outr   r   r   r   r   r{   rR   r[   r   r   rU   )r   r   r   r   r,   )r/   r   r0   r   r^   r   r   r   anyhas_query_strider   conv_cpe_dwnormr   attnr   r   r   r2   rb   r	   rc   )r8   r$   rO   rQ   r{   rR   r   rS   r   r   r   r   r   r   r   r|   r'   rU   rV   rW   r   r   r   r   r   rd   sr9   s                              r    r0   zMobileAttention.__init__  sk   6 	ot-/+J	B1:7):JF
&}5" #D4F4F$GqQU$G H ,.! D  $D"6U;	G#q(((')I-##+#-! ##%DI" $###DI "-+G5KLD!{{}D5C.1q %Hs    Ec                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S r   r   ro   s     r    rq   zMobileAttention.feature_info!  r   r"   c                     | j                   | j                  |      }||z   }|}| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rs   )r   r   r   r   r^   rc   )r8   r?   x_cperu   s       r    rA   zMobileAttention.forward'  su    '$$Q'EE	AIIaLIIaLQ==q!H,Ar"   rv   rK   s   @r    r   r     s    "#$)!'&' #%77$&NN,0$&""6:"!3XWXW XW 	XW
  XW XW XW XW XW XW XW "XW XW XW !$XW  !XW" !#XW$ "%XW& y)'XW( ")XW* +XW, -XW. %-UO/XW0 1XW2 3XWtFr"   r   c            #            e Zd ZdZdddddddddej
                  ej                  dddd	fd
ededededededede	de
dededededee   dee   dede
f" fdZd Z xZS )r   z, Inverted residual block w/ CondConv routingr<   r   rM   Fr   Nr   rN   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   num_expertsrW   c                     || _         t        | j                         }t        t        |   |||||||||	|
|||||||       t        j                  || j                         | _        y )N)r   )r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   r   rW   )r   rm   r/   r   r0   r2   Linear
routing_fn)r8   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   r   rW   r   r9   s                      r    r0   zCondConvResidual.__init__9  s    * 't'7'78.)!+)!#)# 	/ 	
& ))FD,<,<=r"   c                    |}t        j                  |d      j                  d      }t        j                  | j                  |            }| j                  ||      }| j                  |      }| j                  ||      }| j                  |      }| j                  |      }| j                  ||      }| j                  |      }| j                  r| j                  |      |z   }|S )Nr   )Fadaptive_avg_pool2dflattenr   sigmoidr   r   r`   r   r   r   r   r   r^   rc   )r8   r?   ru   pooled_inputsrouting_weightss        r    rA   zCondConvResidual.forwarde  s    --a3;;A>--(FGLLO,HHQKLLO,HHQKGGAJMM!_-HHQK==q!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r0   rA   rJ   rK   s   @r    r   r   6  s    6 #$ "#$"##%77$&NN,0-1 $&%*>*> *>  	*>
 *> *> *> *> *> *> !*>  *> !*> "*> y)*>  z*!*>" #*>$ "%*>Xr"   r   c            !            e Zd ZdZdddddddddej
                  ej                  ddd	fd
edededededededede	de
dedededee   dee   de
f  fdZd Zd Z xZS )r   a(   Residual block with expansion convolution followed by pointwise-linear w/ stride

    Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML`
        - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html

    This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers
      * MobileDet - https://arxiv.org/abs/2004.14525
      * EfficientNet-X - https://arxiv.org/abs/2102.05610
      * EfficientNet-V2 - https://arxiv.org/abs/2104.00298
    r<   r   r   rM   Fr   NrN   r$   rO   r   rQ   rR   r   rS   force_in_chsr|   r   r}   r'   rU   rV   r   rW   c           	      D   t         t        |           t        ||      }|dkD  rt	        ||
z        }nt	        ||
z        }t        ||      }||k(  xr |dk(  xr |	 | _        |d uxr |dkD  }t        ||||rdn||||      | _         ||d      | _	        t        ||||      | _        |r
 |||      nt        j                         | _        t        ||||      | _         ||d	
      | _        |rt#        |      | _        y t        j                         | _        y )Nr   r   rY   Tr-   r\   r   r   Fr   )r/   r   r0   r   r
   r!   r^   r   conv_expr`   r   ra   r2   rb   r   r   r   r	   rc   )r8   r$   rO   r   rQ   rR   r   rS   r   r|   r   r}   r'   rU   rV   r   rW   rd   r   rZ   re   r9   s                        r    r0   zEdgeResidual.__init__  s   & 	lD*,+J	B!$\I%=>G$Vi%78GJ07*:v{JF
%4&1* &G_1Ffh@ "'48HwvfU =E(7i8"++- &gwPXY!'U;5C.1r"   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S r   r   ro   s     r    rq   zEdgeResidual.feature_info  r   r"   c                    |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rs   )r   r`   ra   r   r   r   r^   rc   rt   s      r    rA   zEdgeResidual.forward  sv    MM!HHQKGGAJGGAJMM!HHQK==q!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r0   rq   rA   rJ   rK   s   @r    r   r   u  s   	 $% ! "#$#%77$&NN,0-1$&#,W,W ,W !	,W
 ,W ,W ,W ,W ,W ,W ,W !,W !,W ",W y),W  z*!,W" "#,W\G
r"   r   )%rE   typingr   r   r   r   r   torch.nnr2   r   r   timm.layersr   r	   r
   r   r   r   r   r   r   r   r   __all__Moduler   rH   r!   r   r   r   r   r   r   r   r   r   r   r"   r    <module>r      s    2 1   $H H H H
 "))_
&8C= &C &%#BII %#P-		 -`ORYY Od_ryy _D<299 <H		 HVrbii rj<' <~J299 Jr"   