
    kh@                         d dl mZmZ d dlZd dlmZ d dlmc mZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ  G d d	ej"                        Zy)
    )OptionalTypeN   )maybe_add_mask)use_fused_attn)Mlp)trunc_normal_tf_c                   @    e Zd ZU dZej
                  j                  e   ed<   dddddddddd	d
de	j                  dfdededededee   dedededededededeee	j                         deee	j                         def fdZd Zddeej&                     fdZ xZS ) AttentionPoolLatentz' Attention pooling w/ latent query
    
fused_attnN   g      @TFr    tokeng        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layer	act_layerdropc                 \   t         |           |xs |}|xs |}||z  dk(  sJ || _        ||z  | _        || _        | j                  dz  | _        || _        t               | _        |dk(  r3|J t        j                  t        j                  ||            | _        nd | _        |
xs || _        |	| _        t        j                  t        j                  d| j                  |            | _        t        j"                  |||      | _        t        j"                  ||dz  |      | _        |rC|xs t        j(                  } || j                        | _         || j                        | _        n2t        j.                         | _        t        j.                         | _        t        j"                  ||      | _        t        j2                  |      | _        | ||      nt        j.                         | _        t9        |t;        ||z        |      | _        | j?                          y )Nr         absr   )bias   )r   ) super__init__r   head_dimr   scalepoolr   r   nn	Parametertorchzerosr   r   r   latentLinearqkv	LayerNormq_normk_normIdentityprojDropout	proj_dropnormr   intmlpinit_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   qk_norm_layer	__class__s                    V/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/attention_pool.pyr%   zAttentionPoolLatent.__init__   s   $ 	,	#2{9$)))"!Y."]]d*
	(*(((\\%++i*MNDN!DN$1	$ll5;;q$//9#MN9ih?))Iy1}8D&6",,M'6DK'6DK++-DK++-DKIIi3	D)0:0FJ|,BKKM	y#i)&;"<	R    c                     | j                   1t        | j                   | j                   j                  d   dz         t        | j                  | j                  dz         y )Nr   r    )std)r   r	   shaper-   r   )r<   s    r?   r;   z AttentionPoolLatent.init_weightsJ   sE    >>%T^^1E1Ea1HD1PQ$//T*ABr@   	attn_maskc                    |j                   \  }}}| j                  7|| j                  j                  d      j                  |j                        z   }| j
                  j                  |dd      }| j                  |      j                  || j                  | j                  | j                        j                  dd      }| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }	}
| j!                  |      | j#                  |	      }	}| j$                  rt'        j(                  ||	|
|      }nG|| j*                  z  }||	j                  dd      z  }t-        ||      }|j/                  d	      }||
z  }|j                  dd      j                  || j                  |      }| j1                  |      }| j3                  |      }|| j5                  | j7                  |            z   }| j8                  d
k(  r|d d df   }|S | j8                  dk(  r|j;                  d      }|S )Nr   r   r#         )rD   )dimr   avg)rC   r   	unsqueezetodtyper-   expandr/   reshaper   r   r&   	transposer0   permuteunbindr2   r3   r   Fscaled_dot_product_attentionr'   r   softmaxr5   r7   r:   r8   r(   mean)r<   xrD   BNCq_latentr/   r0   kvattns               r?   forwardzAttentionPoolLatent.forwardO   s   ''1a>>%DNN,,Q/22177;;A;;%%aR0FF8$$QWaabcefgWWQZ1aGOOPQSTVWYZ\]^yy|1{{1~t{{1~1??..q!Q)LADJJAq{{2r**D!$	2D<<B<'DqAKK1%%a!<IIaLNN11&& 99!Q$A  YY%q	Ar@   )N)__name__
__module____qualname____doc__r+   jitFinalbool__annotations__r)   GELUr9   r   floatstrr   Moduler%   r;   Tensorr`   __classcell__)r>   s   @r?   r   r      s   		%%
 !%!'+"!!"$483577!66 6 	6
 6  }6 6 6 6 6 6 6 6 !bii16  RYY06  !6pC
"HU\\$: "r@   r   )typingr   r   r+   torch.nnr)   torch.nn.functional
functionalrT   	attentionr   configr   r:   r   weight_initr	   rl   r    r@   r?   <module>rw      s2    !     % "  )d")) dr@   