
    khT                     T   d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ g dZ G d de
j>                        Z  G d de
j>                        Z! G d de
j>                        Z" G d de
j>                        Z# G d de
j>                        Z$d5dZ%d6dZ&d7dZ' e e'ddd       e'dd       e'dd d       e'dd!       e'dd"       e'dd#d       e'dd$       e'dd%       e'dd&       e'dd'd(      d)
      Z(ed6d*e$fd+       Z)ed6d*e$fd,       Z*ed6d*e$fd-       Z+ed6d*e$fd.       Z,ed6d*e$fd/       Z-ed6d*e$fd0       Z.ed6d*e$fd1       Z/ed6d*e$fd2       Z0ed6d*e$fd3       Z1ed6d*e$fd4       Z2y)8a3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partial)ListOptionalTupleUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPathtrunc_normal_use_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)register_modelgenerate_default_cfgs)Cait	ClassAttnLayerScaleBlockClassAttnLayerScaleBlockTalkingHeadAttnc                   `     e Zd ZU ej                  j
                  e   ed<   d fd	Zd Z	 xZ
S )r   
fused_attnc                    t         |           || _        ||z  }|dz  | _        t	               | _        t        j                  |||      | _        t        j                  |||      | _	        t        j                  |||      | _
        t        j                  |      | _        t        j                  ||      | _        t        j                  |      | _        y )N      ࿩bias)super__init__	num_headsscaler   r   nnLinearqkvDropout	attn_dropproj	proj_dropselfdimr$   qkv_biasr,   r.   head_dim	__class__s          L/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/cait.pyr#   zClassAttn.__init__    s    ")#%
(*3(33(33(3I.IIc3'	I.    c                    |j                   \  }}}| j                  |d d df         j                  d      j                  |d| j                  || j                  z        j                  dddd      }| j                  |      j                  ||| j                  || j                  z        j                  dddd      }| j                  |      j                  ||| j                  || j                  z        j                  dddd      }| j                  rPt        j                  j                  j                  |||| j                  r| j                  j                  nd      }nL|| j                   z  }||j#                  dd      z  }	|	j%                  d	      }	| j                  |	      }	|	|z  }|j#                  dd      j                  |d|      }| j'                  |      }| j)                  |      }|S )
Nr   r                 )	dropout_pr1   )shaper(   	unsqueezereshaper$   permuter)   r*   r   torchr&   
functionalscaled_dot_product_attentiontrainingr,   pr%   	transposesoftmaxr-   r.   )
r0   xBNCr(   r)   r*   x_clsattns
             r5   forwardzClassAttn.forward.   s   ''1aFF1QT7O%%a(00At~~qDNNGZ[ccdeghjkmnoFF1IaDNNA4GHPPQRTUWXZ[\FF1IaDNNA4GHPPQRTUWXZ[\??HH''DD1a.2mm$..** E E
 DJJAq{{2r**D<<B<'D>>$'D1HE1%--aA6		% u%r6      Fr:   r:   )__name__
__module____qualname__rC   jitFinalbool__annotations__r#   rP   __classcell__r4   s   @r5   r   r      s"     		%%/r6   r   c            
       b     e Zd Zdddddej                  ej
                  eedf
 fd	Zd Z	 xZ
S )r         @Fr:   -C6?c                    t         |            |	|      | _         |
|||||      | _        |dkD  rt	        |      nt        j                         | _         |	|      | _        t        ||z        } |||||      | _
        t        j                  |t        j                  |      z        | _        t        j                  |t        j                  |      z        | _        y N)r$   r2   r,   r.   r:   )in_featureshidden_features	act_layerdropr"   r#   norm1rO   r   r&   Identity	drop_pathnorm2intmlp	ParameterrC   onesgamma_1gamma_2r0   r1   r$   	mlp_ratior2   r.   r,   rh   rc   
norm_layer
attn_block	mlp_blockinit_valuesmlp_hidden_dimr4   s                 r5   r#   z!LayerScaleBlockClassAttn.__init__J        	_

	 1:B),BKKM_
S9_-*	
 ||K%**S/$AB||K%**S/$ABr6   c           	      4   t        j                  ||fd      }|| j                  | j                  | j	                  | j                  |            z        z   }|| j                  | j                  | j                  | j                  |            z        z   }|S )Nr   r>   )	rC   catrh   rn   rO   rf   ro   rk   ri   )r0   rJ   rN   us       r5   rP   z LayerScaleBlockClassAttn.forwardn   sv    IIuaja(t||dii

16N'NOOt||dhhtzz%?P6Q'QRRr6   )rS   rT   rU   r&   GELU	LayerNormr   r   r#   rP   rZ   r[   s   @r5   r   r   G   s7     gg|| "CHr6   r   c                   &     e Zd Zd fd	Zd Z xZS )r   c                    t         |           || _        ||z  }|dz  | _        t	        j
                  ||dz  |      | _        t	        j                  |      | _        t	        j
                  ||      | _	        t	        j
                  ||      | _
        t	        j
                  ||      | _        t	        j                  |      | _        y )Nr   r9   r    )r"   r#   r$   r%   r&   r'   qkvr+   r,   r-   proj_lproj_wr.   r/   s          r5   r#   zTalkingHeadAttn.__init__x   s    ")#%
99S#'9I.IIc3'	ii	95ii	95I.r6   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   | j
                  z  |d   |d   }}}||j                  dd      z  }	| j                  |	j	                  dddd            j	                  dddd      }	|	j                  d      }	| j                  |	j	                  dddd            j	                  dddd      }	| j                  |	      }	|	|z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr9   r8   r   r      r<   r=   r>   )r?   r   rA   r$   rB   r%   rH   r   rI   r   r,   r-   r.   )
r0   rJ   rK   rL   rM   r   r(   r)   r*   rO   s
             r5   rP   zTalkingHeadAttn.forward   sL   ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdefa&4::%s1vs1va11;;r2&&{{4<<1a34<<Q1aH|||#{{4<<1a34<<Q1aH~~d#AX  A&..q!Q7IIaLNN1r6   rQ   )rS   rT   rU   r#   rP   rZ   r[   s   @r5   r   r   u   s    /&r6   r   c            
       b     e Zd Zdddddej                  ej
                  eedf
 fd	Zd Z	 xZ
S )r   r]   Fr:   r^   c                    t         |            |	|      | _         |
|||||      | _        |dkD  rt	        |      nt        j                         | _         |	|      | _        t        ||z        } |||||      | _
        t        j                  |t        j                  |      z        | _        t        j                  |t        j                  |      z        | _        y r`   re   rp   s                 r5   r#   zLayerScaleBlock.__init__   rw   r6   c           	         || j                  | j                  | j                  | j                  |            z        z   }|| j                  | j                  | j                  | j                  |            z        z   }|S N)rh   rn   rO   rf   ro   rk   ri   r0   rJ   s     r5   rP   zLayerScaleBlock.forward   sa    t||dii

1.FFGGt||dhhtzz!}.EEFFr6   )rS   rT   rU   r&   r{   r|   r   r   r#   rP   rZ   r[   s   @r5   r   r      s7     gg||&"CHr6   r   c                       e Zd Zdddddddddd	d
d
d
d
d
eee eej                  d      ej                  e
edeeddf fd	Zd Zej                   j"                  d        Zej                   j"                  d'd       Zej                   j"                  d(d       Zej                   j"                  dej*                  fd       Zd)dedee   fdZ	 	 	 	 	 d*dej6                  deeeee   f      dededededeeej6                     eej6                  eej6                     f   f   fdZ 	 	 	 d+deeee   f   d ed!efd"Z!d# Z"d(d$efd%Z#d& Z$ xZ%S ),r         r9     token      r]   Tr:   ư>)epsr^   r8   c                 &   t         !|           |dv sJ || _        || _        |x| _        x| _        | _        d| _         |||||      | _        | j                  j                  }t        | j                  d      r| j                  j                         n|}t        j                  t        j                  dd|            | _        t        j                  t        j                  d||            | _        t        j$                  |      | _        t)        |      D cg c]  }| }}t        j*                  t)        |      D cg c]  } ||||	|
||||   |||||       c} | _        t)        |      D cg c]  }t/        ||d| 	       c}| _        t        j2                  t)        |      D  cg c]  }  |||||
|||||
	       c}       | _         ||      | _        t        j$                  |      | _        |dkD  rt        j:                  ||      nt        j<                         | _        tA        | j"                  d       tA        | j                   d       | jC                  | jD                         y c c}w c c}w c c}w c c} w )N r   avgF)img_size
patch_sizein_chans	embed_dim
feat_ratior   )rG   )r1   r$   rq   r2   r.   r,   rh   rr   rc   rs   rt   ru   blocks.)num_chs	reductionmodule)	r1   r$   rq   r2   rr   rc   rs   rt   ru   r   {Gz?std)#r"   r#   num_classesglobal_poolnum_featureshead_hidden_sizer   grad_checkpointingpatch_embednum_patcheshasattrr   r&   rl   rC   zeros	cls_token	pos_embedr+   pos_droprange
Sequentialblocksdictfeature_info
ModuleListblocks_token_onlynorm	head_dropr'   rg   headr   apply_init_weights)"r0   r   r   r   r   r   r   depthr$   rq   r2   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_rateblock_layersblock_layers_tokenpatch_layerrr   rc   rs   rt   ru   attn_block_token_onlymlp_block_token_onlydepth_token_onlymlp_ratio_token_onlyr   ridpr_r4   s"                                    r5   r#   zCait.__init__   s_   < 	2222&&ENNND1DN"'&!	
 &&22-4T5E5E|-TD'')Zdekk!Q	&BCekk![)&LM

]3',U|4!~44mm <&!  '3$$!f!!#'
 &! " bgglamn\]T)q7STRUWn!# )*
0,  1C*!,*#
1
 
0, 
"- y)	I.9DqBIIi5bkkm	dnn#.dnn#.

4%%&I 5&! o
0,s   	I?J:J	6Jc                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r         ?)	
isinstancer&   r'   r   weightr!   init	constant_r|   )r0   ms     r5   r   zCait._init_weights(  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r6   c                 
    ddhS )Nr   r    r0   s    r5   no_weight_decayzCait.no_weight_decay1  s    [))r6   c                     || _         y r   )r   )r0   enables     r5   set_grad_checkpointingzCait.set_grad_checkpointing5  s
    "(r6   c                       fd}|S )Nc                    t        dD cg c]  }| j                  |       c}      ry| j                  d      r t        | j                  d      d         dz   S | j                  d      rNt	        j
                        t	        j                        z
  dz   }t        | j                  d      d         |z   S | j                  d      rt	        j
                        S t        d      S c c}w )	N)r   r   r   r   r   .r   zblocks_token_only.znorm.inf)any
startswithrj   splitlenr   r   float)namen	to_offsetr0   s      r5   _matcherz$Cait.group_matcher.<locals>._matcher;  s    0YZ1DOOA&Z[+4::c?1-.22!56,s43I3I/JJQN	4::c?1-.::)4;;''U|# [s   C,r   )r0   coarser   s   `  r5   group_matcherzCait.group_matcher9  s    	$ r6   returnc                     | j                   S r   )r   r   s    r5   get_classifierzCait.get_classifierJ  s    yyr6   r   r   c                     || _         ||dv sJ || _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   r   )r   r   r&   r'   r   rg   r   )r0   r   r   s      r5   reset_classifierzCait.reset_classifierN  sU    &""6666*DALqBIId//=	VXVaVaVc	r6   rJ   indicesr   
stop_early
output_fmtintermediates_onlyc           	      6   |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
|j                  \  }}}}| j	                  |      }|| j
                  z   }| j                  |      }t        j                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]m  \  }}| j                  r+t        j                  j                         st        ||      }n ||      }||	v sJ|j                  |r| j                  |      n|       o |ra| j                  j                  ||f      \  }}|D cg c]6  }|j!                  |||d      j#                  dddd	      j%                         8 }}|r|S | j&                  j)                  |j                  d   dd      }t        | j*                        D ]  \  }} |||      } t        j,                  ||fd
      }| j                  |      }||fS c c}w )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        )NCHWNLCz)Output format must be one of NCHW or NLC.r   Nr   r=   r   r9   r8   r>   )r   r   r   r?   r   r   r   rC   rV   is_scripting	enumerater   r   appendr   dynamic_feat_sizerA   rB   
contiguousr   expandr   ry   )r0   rJ   r   r   r   r   r   rA   intermediatestake_indices	max_indexrK   r   heightwidthr   r   blkHWy
cls_tokenss                         r5   forward_intermediateszCait.forward_intermediatesU  s    & _,Y.YY,&"6s4;;7G"Qi  gg1feQMM!99!!#:[[F[[)a-0F' 	BFAs&&uyy/E/E/GsA&FL $$TTYYq\qA	B ##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMl   ^^**1771:r2>
 6 67 	,FAsQ
+J	,IIz1o1-IIaL- ms   ;H
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r+t        j                         | _        | j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )	r   r   r   r&   rg   r   r   r   r   )r0   r   r  r  r   r   s         r5   prune_intermediate_layerszCait.prune_intermediate_layers  sk     #7s4;;7G"Qikk.9q=1DI%']]_D"!!!R(r6   c                     | j                  |      }|| j                  z   }| j                  |      }| j                  r5t        j
                  j                         st        | j                  |      }n| j                  |      }| j                  j                  |j                  d   dd      }t        | j                        D ]  \  }} |||      } t	        j                  ||fd      }| j                  |      }|S )Nr   r=   r   r>   )r   r   r   r   rC   rV   r   r   r   r   r   r?   r   r   ry   r   )r0   rJ   r  r   r   s        r5   forward_featureszCait.forward_features  s    QMM!""599+A+A+Ct{{A.AAA^^**1771:r2>
 6 67 	,FAsQ
+J	,IIz1o1-IIaLr6   
pre_logitsc                     | j                   r3| j                   dk(  r|d d dd f   j                  d      n|d d df   }| j                  |      }|r|S | j                  |      S )Nr   r   r>   r   )r   meanr   r   )r0   rJ   r  s      r5   forward_headzCait.forward_head  s_    (,(8(8E(A!QR%!$qAwANN1q0DIIaL0r6   c                 J    | j                  |      }| j                  |      }|S r   )r
  r  r   s     r5   rP   zCait.forward  s'    !!!$a r6   )TFr   )NFFr   F)r   FT)&rS   rT   rU   r   r   r   r   r&   r|   r{   r   r   r   r#   r   rC   rV   ignorer   r   r   Moduler   rj   r   strr   Tensorr   r   rX   r   r  r  r
  r  rP   rZ   r[   s   @r5   r   r      s   
 (7"r||6gg&"+!$!$9W'r- YY* * YY) ) YY   YY		  dC dhsm d 8<$$',; ||;  eCcN34;  	; 
 ;  ;  !%;  
tELL!5tELL7I)I#JJ	K; ~ ./$#	3S	>*  	"1$ 1r6   r   c                 v    d| v r| d   } i }| j                         D ]  \  }}|||j                  dd      <    |S )Nmodelzmodule.r   )itemsreplace)
state_dictr  checkpoint_no_moduler)   r*   s        r5   checkpoint_filter_fnr    sR    *(
  " ;19:QYYy"56;r6   c                 r    |j                  dd      }t        t        | |ft        t	        |d      d|}|S )Nout_indicesr9   getter)r  feature_cls)pretrained_filter_fnfeature_cfg)popr   r   r  r   )variant
pretrainedkwargsr  r  s        r5   _create_caitr&    sJ    **]A.K  2[hG E Lr6   c                 2    | ddd dddt         t        ddd|S )	Nr   )r9     r(  r   bicubicTzpatch_embed.projr   )urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  r   
first_conv
classifierr   )r*  r%  s     r5   _cfgr2    s2    =t)%.B(  r6   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r9   r   r   )	hf_hub_idr*  r+  z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)r3  r*  z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r9     r4  )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1kr   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr         r   h㈵>r   r   r   r$   ru   r$  )cait_xxs24_224r   r&  r$  r%  
model_argsr  s       r5   r:  r:    7    s"W[\J_j_DD^W]D^_ELr6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r6  r7  r   r8  r9  r$  )cait_xxs24_384r;  r<  s       r5   r@  r@    r>  r6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r6  $   r   r8  r9  r$  )cait_xxs36_224r;  r<  s       r5   rC  rC    r>  r6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r6  rB  r   r8  r9  r$  )cait_xxs36_384r;  r<  s       r5   rE  rE  $  r>  r6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   i   r7     r8  r9  r$  )cait_xs24_384r;  r<  s       r5   rH  rH  +  s7    s"W[\J^Z^4
C]V\C]^ELr6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r(  r7  rR   r8  r9  r$  )cait_s24_224r;  r<  s       r5   rJ  rJ  2  7    s"W[\J]J]$zB\U[B\]ELr6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r(  r7  rR   r8  r9  r$  )cait_s24_384r;  r<  s       r5   rM  rM  9  rK  r6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r(  rB  rR   r   r9  r$  )cait_s36_384r;  r<  s       r5   rO  rO  @  rK  r6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )Nr   r   rB  r   r9  r$  )cait_m36_384r;  r<  s       r5   rQ  rQ  G  7    s"X\]J]J]$zB\U[B\]ELr6   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )Nr   r   0   r   r9  r$  )cait_m48_448r;  r<  s       r5   rU  rU  N  rR  r6   r   r  )r   )3__doc__	functoolsr   typingr   r   r   r   rC   torch.nnr&   	timm.datar	   r
   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__r  r   r   r   r   r   r  r&  r2  default_cfgsr:  r@  rC  rE  rH  rJ  rM  rO  rQ  rU  r   r6   r5   <module>rb     sf    / /   A P P * + 3 <
a)		 )X+ryy +\'bii 'T*bii *Zo299 od 
 %#'? $
 $(?$ $(? $
 $(?$ #'># "&= "
 "&=" "&=" "&=" "&= "Q-& -` $   $   $   $                    r6   