
    khL                        d Z ddlZddlmZmZmZmZmZ ddlZddl	m
Z
 ddlm
c mZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZm Z  dgZ! G d de
jD                        Z# G d de
jD                        Z$ G d de
jD                        Z% G d de
jD                        Z& G d de
jD                        Z' G d de
jD                        Z(d Z)d%dZ*d&dZ+ e  e+d       e+d       e+d       e+d       e+d       e+d       e+d      d      Z,ed%de(fd       Z-ed%de(fd       Z.ed%de(fd        Z/ed%de(fd!       Z0ed%de(fd"       Z1ed%de(fd#       Z2ed%de(fd$       Z3y)'a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
    N)CallableListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tuple	to_ntupletrunc_normal_	LayerNormuse_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpoint)register_modelgenerate_default_cfgsPyramidVisionTransformerV2c                   P     e Zd Zddej                  ddf fd	Zdee   fdZ xZ	S )MlpWithDepthwiseConvN        Fc           	         t         |           |xs |}|xs |}t        j                  ||      | _        |rt        j
                         nt        j                         | _        t        j                  ||dddd|      | _	         |       | _
        t        j                  ||      | _        t        j                  |      | _        y )N   r   T)biasgroups)super__init__nnLinearfc1ReLUIdentityreluConv2ddwconvactfc2Dropoutdrop)selfin_featureshidden_featuresout_features	act_layerr,   
extra_relu	__class__s          N/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/pvt_v2.pyr    zMlpWithDepthwiseConv.__init__$   s     	#2{)8[99[/:!+BGGI	ii!QPT]lm;99_l;JJt$	    	feat_sizec                    | j                  |      }|j                  \  }}}|j                  dd      j                  |||d   |d         }| j	                  |      }| j                  |      }|j                  d      j                  dd      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S )Nr      r   )
r#   shape	transposeviewr&   r(   flattenr)   r,   r*   )r-   xr6   BNCs         r4   forwardzMlpWithDepthwiseConv.forward7   s    HHQK''1aKK1""1a1y|DIIaLKKNIIaL""1a(HHQKIIaLHHQKIIaLr5   )
__name__
__module____qualname__r!   GELUr    r   intrA   __classcell__r3   s   @r4   r   r   #   s-     !gg%&DI r5   r   c                   x     e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 d fd	Zde	e
   fdZ xZS )	Attention
fused_attnc                 z   t         |           ||z  dk(  sJ d| d| d       || _        || _        ||z  | _        | j                  dz  | _        t               | _        t        j                  |||      | _
        t        j                  ||dz  |      | _        t        j                  |      | _        t        j                  ||      | _        t        j                  |      | _        |sbd | _        |dkD  r@t        j"                  ||||	      | _        t        j&                  |      | _        d | _        y d | _        d | _        d | _        y t        j,                  d
      | _        t        j"                  ||dd	      | _        t        j&                  |      | _        t        j.                         | _        y )Nr   zdim z  should be divided by num_heads .g      )r   r8   r   )kernel_sizestride   )r   r    dim	num_headshead_dimscaler   rK   r!   r"   qkvr+   	attn_dropproj	proj_droppoolr'   srr   normr)   AdaptiveAvgPool2drE   )	r-   rQ   rR   sr_ratiolinear_attnqkv_biasrW   rY   r3   s	           r4   r    zAttention.__init__H   sh    	Y!#]tC50PQZP[[\%]]#"y(]]d*
(*3(3))Cqx8I.IIc3'	I.DI!|))C(8TLL-	 DH  	DH,,Q/DIiiSaBDGS)DIwwyDHr5   r6   c                 d   |j                   \  }}}|\  }}| j                  |      j                  ||| j                  d      j	                  dddd      }| j
                  |j	                  ddd      j                  ||||      }| j                  | j                  |            j                  ||d      j	                  ddd      }| j                  |      }| j                  |      }| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	n| j                  |j	                  ddd      j                  ||||      }| j                  |      j                  ||d      j	                  ddd      }| j                  |      }| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	nK| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	|	j                  d      \  }
}| j                  r<t        j                  ||
|| j                  r| j                   j"                  nd      }nL|| j$                  z  }||
j'                  d	d      z  }|j)                  d
      }| j!                  |      }||z  }|j'                  dd      j                  |||      }| j+                  |      }| j-                  |      }|S )Nr   r8   r   r      r   )	dropout_prQ   )r9   rU   reshaperR   permuterZ   r[   r\   r)   rV   rS   unbindrK   Fscaled_dot_product_attentiontrainingrW   prT   r:   softmaxrX   rY   )r-   r=   r6   r>   r?   r@   HWrU   rV   kvattns                r4   rA   zAttention.forwardp   s   ''1a1FF1IaDNNB7??1aK99 		!Q"**1aA6A		!%--aB7??1aHA		!AA##Ar1dnndmmLTTUVXY[\^_abcBww"IIaA&..q!Q:GGAJ&&q!R088AqAIIaLWWQZ''2q$..$--PXXYZ\]_`bcefgWWQZ''2q$..$--PXXYZ\]_`bcefgyy|1??..q!QVZVcVc$..BRBRiklADJJAq{{2r**D<<B<'D>>$'DqAKK1%%aA.IIaLNN1r5   )   r   FTr   r   )rB   rC   rD   torchjitFinalbool__annotations__r    r   rF   rA   rG   rH   s   @r4   rJ   rJ   E   s@    		%%
 &!P!DI !r5   rJ   c            	       X     e Zd Zdddddddej                  ef	 fd	Zdee   fdZ	 xZ
S )Block      @r   Fr   c           	      v   t         |            ||      | _        t        |||||||      | _        |	dkD  rt        |	      nt        j                         | _         ||      | _	        t        |t        ||z        |
||      | _        |	dkD  rt        |	      | _        y t        j                         | _        y )N)rR   r^   r_   r`   rW   rY   r   )r.   r/   r1   r,   r2   )r   r    norm1rJ   rs   r   r!   r%   
drop_path1norm2r   rF   mlp
drop_path2)r-   rQ   rR   	mlp_ratior^   r_   r`   rY   rW   	drop_pathr1   
norm_layerr3   s               r4   r    zBlock.__init__   s     	_
#
	 2;R(9-R[[]_
'i0"
 2;R(9-R[[]r5   r6   c                     || j                  | j                  | j                  |      |            z   }|| j                  | j	                  | j                  |      |            z   }|S N)r   rs   r~   r   r   r   )r-   r=   r6   s      r4   rA   zBlock.forward   sS    		$**Q- CDDA	 BCCr5   )rB   rC   rD   r!   rE   r   r    r   rF   rA   rG   rH   s   @r4   r{   r{      s;     gg #SJDI r5   r{   c                   *     e Zd ZdZd fd	Zd Z xZS )OverlapPatchEmbedz Image to Patch Embedding
    c           	          t         |           t        |      }t        |      |kD  sJ d       || _        t        j                  |||||d   dz  |d   dz  f      | _        t        j                  |      | _	        y )Nz!Set larger patch_size than strider   r8   r   )rO   padding)
r   r    r   max
patch_sizer!   r'   rX   r   r\   )r-   r   rO   in_chans	embed_dimr3   s        r4   r    zOverlapPatchEmbed.__init__   s}    z*
:'L)LL'$IIiJqMQ$6
18J#KM	 LL+	r5   c                 r    | j                  |      }|j                  dddd      }| j                  |      }|S )Nr   r8   r   r   )rX   rh   r\   r-   r=   s     r4   rA   zOverlapPatchEmbed.forward   s5    IIaLIIaAq!IIaLr5   )rP   rc   r   i   )rB   rC   rD   __doc__r    rA   rG   rH   s   @r4   r   r      s    ,r5   r   c                        e Zd Zdddddddddef
deded	ed
ededededededededeee   ef   de	f fdZ
d Z xZS )PyramidVisionTransformerStageTrt   r   Fr|   r   rQ   dim_outdepth
downsamplerR   r^   r_   r   r`   rY   rW   r   r   c                 P   t         |           d| _        |rt        dd||      | _        n||k(  sJ d | _        t        j                  t        |      D cg c],  }t        ||||||	|
|t        |t              r||   n||
      . c}      | _         ||      | _        y c c}w )NFr   r8   r   rO   r   r   )
rQ   rR   r^   r_   r   r`   rY   rW   r   r   )r   r    grad_checkpointingr   r   r!   
ModuleListranger{   
isinstancelistblocksr\   )r-   rQ   r   r   r   rR   r^   r_   r   r`   rY   rW   r   r   ir3   s                  r4   r    z&PyramidVisionTransformerStage.__init__   s      	"'/!	DO '>!>"DOmm <%!  &+#&0D&Aily!&
 %! " w'	%!s   1B#c                    | j                   | j                  |      }|j                  \  }}}}||f}|j                  |d|      }| j                  D ]C  }| j                  r,t
        j                  j                         st        |||      }; |||      }E | j                  |      }|j                  ||d   |d   d      j                  dddd      j                         }|S )Nrb   r   r   r   r8   )r   r9   rg   r   r   ru   rv   is_scriptingr   r\   rh   
contiguous)r-   r=   r>   ro   rp   r@   r6   blks           r4   rA   z%PyramidVisionTransformerStage.forward  s    ??&"AWW
1aF	IIaQ;; 	&C&&uyy/E/E/GsAy19%		&
 IIaLIIa1y|R8@@Aq!LWWYr5   )rB   rC   rD   r   rF   rx   floatr   r   r   r    rA   rG   rH   s   @r4   r   r      s      $ %"!!!36#,+(+( +( 	+(
 +( +( +( +( +( +( +( +( T%[%/0+( !+(Zr5   r   c                   *    e Zd Zddddddddd	d
ddddef fd	Zd Zd Zej                  j                  d        Z
ej                  j                  d%d       Zej                  j                  d&d       Zdej                  fdZd'dedee   fdZ	 	 	 	 	 d(dej(                  deeeee   f      dededededeeej(                     eej(                  eej(                     f   f   fdZ	 	 	 d)deeee   f   dedefd Zd! Zd%d"efd#Zd$ Z xZS )*r   r     avgr   rc      r   )@            )r   r8   rc   rt   )rt   rc   r8   r   )       @r   r|   r|   TFr   c                    t         |           || _        |dv sJ || _        || _        t        |      } t        |      |      } t        |      |      } t        |      |      }t        |      |k(  sJ g | _        t        dd||d         | _	        t        j                  d|t        |            j                  |      D cg c]  }|j                          }}d}|d   }g }t        |      D ]k  }|t!        |||   ||   |dkD  ||   ||   ||   |
|	||||   |      gz  }||   }|||   z  }| xj                  t#        |dd|z  z  d| 	      gz  c_        m t%        j&                  | | _        |d
   x| _        | _        t%        j.                  |      | _        |dkD  rt%        j2                  |d
   |      nt%        j4                         | _        | j9                  | j:                         y c c}w )Nr    rP   rc   r   r   )rQ   r   r   r   rR   r^   r   r_   r`   rY   rW   r   r   r8   stages.)num_chs	reductionmodulerb   )r   r    num_classesglobal_pooldepthslenr   feature_infor   patch_embedru   linspacesumsplittolistr   r   dictr!   
Sequentialstagesnum_featureshead_hidden_sizer+   	head_dropr"   r%   headapply_init_weights)r-   r   r   r   r   
embed_dimsrR   	sr_ratios
mlp_ratiosr`   linear	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   
num_stagesr=   dprcurprev_dimr   r   r3   s                          r4   r    z#PyramidVisionTransformerV2.__init__  s#   $ 	&k)))&[
*Yz*:6
)Ij))4	)Ij))4	::---, m	
 $)>>!^S[#Q#W#WX^#_`aqxxz``a=z" 	dA4"1Qiq5#A,"1$Q-"!((a&%  F "!}H6!9C$x1q!t8V]^_]`Ta"b!cc%	d& mmV, 5?rNBD1I.>IAoBIIjnk:SUS^S^S`	

4%%&= as   G8c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                  d|z               |j
                  %|j
                  j                  j!                          y y y )Ng{Gz?)stdr   r   g       @)r   r!   r"   r   weightr   init	constant_r'   rN   out_channelsr   datanormal_mathsqrtzero_)r-   mfan_outs      r4   r   z(PyramidVisionTransformerV2._init_weights\  s    a#!((,!RYY'AFF,>!!!&&!, -?'299%mmA&q)99ANNJG GHHMM!!!TYYsW}%=>vv!!!# "	 &r5   c                 &    d| j                   _        y )NF)r   requires_gradr-   s    r4   freeze_patch_embz+PyramidVisionTransformerV2.freeze_patch_embh  s    ).&r5   c                     i S r    r   s    r4   no_weight_decayz*PyramidVisionTransformerV2.no_weight_decayk  s    	r5   c                      t        dd      }|S )Nz^patch_embedz^stages\.(\d+))stemr   )r   )r-   coarsematchers      r4   group_matcherz(PyramidVisionTransformerV2.group_matchero  s     $
 r5   c                 4    | j                   D ]	  }||_         y r   )r   r   )r-   enabless      r4   set_grad_checkpointingz1PyramidVisionTransformerV2.set_grad_checkpointingw  s     	*A#)A 	*r5   returnc                     | j                   S r   )r   r   s    r4   get_classifierz)PyramidVisionTransformerV2.get_classifier|  s    yyr5   r   r   c                     || _         ||dv sJ || _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   r   )r   r   r!   r"   r   r%   r   )r-   r   r   s      r4   reset_classifierz+PyramidVisionTransformerV2.reset_classifier  sT    &"+---*DALqBIId//=	VXVaVaVc	r5   r=   indicesr\   
stop_early
output_fmtintermediates_onlyc                 r   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]#  \  }} ||      }||v s|j                  |       % |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )	r   r   r   r   ru   rv   r   	enumerateappend)r-   r=   r   r\   r   r   r   intermediatestake_indices	max_indexr   feat_idxstages                r4   forward_intermediatesz0PyramidVisionTransformerV2.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qi Q99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
   -r5   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r   )r-   r   r  r  r  r  s         r4   prune_intermediate_layersz4PyramidVisionTransformerV2.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r5   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r   s     r4   forward_featuresz+PyramidVisionTransformerV2.forward_features  s$    QKKNr5   
pre_logitsc                     | j                   r|j                  d      }| j                  |      }|r|S | j                  |      S )N)rb   re   rf   )r   meanr   r   )r-   r=   r  s      r4   forward_headz'PyramidVisionTransformerV2.forward_head  s>    8$ANN1q0DIIaL0r5   c                 J    | j                  |      }| j                  |      }|S r   )r  r  r   s     r4   rA   z"PyramidVisionTransformerV2.forward  s'    !!!$a r5   F)Tr   )NFFr   F)r   FT)rB   rC   rD   r   r    r   r   ru   rv   ignorer   r   r   r!   Moduler   rF   r   strr   Tensorr   r   rx   r   r  r	  r  r  rA   rG   rH   s   @r4   r   r     s    *""' !C'J
$/ YY  YY  YY* *		 dC dhsm d 8<$$',( ||(  eCcN34(  	( 
 (  (  !%(  
tELL!5tELL7I)I#JJ	K( X ./$#	3S	>*  	
1$ 1r5   c                 ~   d| v r| S i }ddl }| j                         D ]  \  }}|j                  d      rH|j                  dd      }|j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j	                  dd |      }|j	                  dd |      }|||<    |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   Nr   patch_embed1patch_embed2zstages.1.downsamplepatch_embed3zstages.2.downsamplepatch_embed4zstages.3.downsamplezdwconv.dwconvr(   zblock(\d+).(\d+)c                 f    dt        | j                  d            dz
   d| j                  d       S )Nr   r   z.blocks.r8   rF   groupr=   s    r4   <lambda>z&checkpoint_filter_fn.<locals>.<lambda>  s4    GC
Oa<O;PPXYZY`Y`abYcXd2e r5   z
^norm(\d+)c                 D    dt        | j                  d            dz
   dS )Nr   r   z.normr  r  s    r4   r  z&checkpoint_filter_fn.<locals>.<lambda>  s!    gc!''!*o6I5J%,P r5   )reitems
startswithreplacesub)
state_dictmodelout_dictr!  rq   rr   s         r4   checkpoint_filter_fnr)    s     J.H  " 	1<<&		.-8A		.*?@A		.*?@A		.*?@AIIox0FF&(eghiFF="PRST	 Or5   c                     t        t        d            }|j                  d|      }t        t        | |ft
        t        d|      d|}|S )Nrc   out_indicesT)flatten_sequentialr+  )pretrained_filter_fnfeature_cfg)tupler   popr   r   r)  r   )variant
pretrainedkwargsdefault_out_indicesr+  r'  s         r4   _create_pvt2r5    sY    a/**],?@K " 2DkJ E Lr5   c                 2    | dddddt         t        dddd	|S )
Nr   )r      r7  )rP   rP   g?bicubiczpatch_embed.projr   F)urlr   
input_size	pool_sizecrop_pctinterpolationr  r   
first_conv
classifierfixed_input_sizer   )r9  r3  s     r4   _cfgrA    s3    4}SY)%.B(TY	
  r5   ztimm/)	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1kr   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr8   r8   r8   r8   )    r      r   r   r8      rt   r   r   rR   r2  )	pvt_v2_b0r   r5  r2  r3  
model_argss      r4   rJ  rJ    s0    \6HT`aJY
Yd:>XQW>XYYr5   c           	      L    t        ddd      }t        dd| it        |fi |S )NrD  r   r   i@  r   rG  rI  r2  )	pvt_v2_b1rK  rL  s      r4   rP  rP    0    \6IUabJY
Yd:>XQW>XYYr5   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr   rO  rG  rI  r2  )	pvt_v2_b2rK  rL  s      r4   rS  rS    rQ  r5   c           	      L    t        ddd      }t        dd| it        |fi |S )N)r   rc      r   rO  rG  rI  r2  )	pvt_v2_b3rK  rL  s      r4   rV  rV    0    ]7JVbcJY
Yd:>XQW>XYYr5   c           	      L    t        ddd      }t        dd| it        |fi |S )N)r   rt      r   rO  rG  rI  r2  )	pvt_v2_b4rK  rL  s      r4   rZ  rZ    rW  r5   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   r   (   r   rO  rG  )rc   rc   rc   rc   )r   r   rR   r   r2  )	pvt_v2_b5rK  rL  s      r4   r]  r]  $  s6    )<amoJY
Yd:>XQW>XYYr5   c           	      N    t        dddd      }t        dd| it        |fi |S )Nr   rO  rG  T)r   r   rR   r   r2  )pvt_v2_b2_lirK  rL  s      r4   r_  r_  +  s6    (;|\`bJ\:\jA[TZA[\\r5   r  )r   )4r   r   typingr   r   r   r   r   ru   torch.nnr!   torch.nn.functional
functionalrj   	timm.datar	   r
   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r   rJ   r{   r   r   r   r)  r5  rA  default_cfgsrJ  rP  rS  rV  rZ  r]  r_  r   r5   r4   <module>rl     s  "  9 9     A ` ` * + # <'
(299 DL		 L^+BII +\		 (=BII =@v vr( %W-W-W-W-W-W-0&  Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z ]0J ] ]r5   