
    kh-                         d dl mZ d dlZd dlmZ d dlmZmZ d dlmZ d Z G d dej                        Z	 G d	 d
ej                        Z
y)    )OptionalN)nn)r   Tensor)_get_activation_fnc                 r   t        | d      rRt        | d      rFt        j                         | _        | `| j
                  }| j                  }t        ||      | _        nt        | d      rRt        | d      rFt        j                         | _        | `| j
                  }| j                  }t        ||      | _        ngd| j                         v sd| j                         v r-| `
| j
                  }| j                  }t        ||      | _
        nt        d       t        d       t        | d	      rd
| _        | S )Nglobal_poolfc)num_classesinitial_num_features
classifierRegNetTResNetz;Model code-writing is not aligned currently with ml-decoder	drop_rater   )hasattrr   Identityr   r	   r
   num_features	MLDecoderr   	_get_nameheadprintexitr   )modelr
   r   s      R/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/ml_decoder.pyadd_ml_decoder_headr   	   s   um$)=KKMH''))<X		&75,+GKKM''))$S_`	U__&	&)u7H*HJ''));\Z
KLRuk"L    c                   x     e Zd Z	 	 d	 d fdZ fdZ	 	 	 	 ddededee   dee   dee   d	ee   defd
Z xZS )TransformerDecoderLayerOptimalreturnc                 h   t         t        |           t        j                  ||      | _        t        j                  |      | _        t        j                  |      | _        t        j                  |      | _	        t        j                  |      | _
        t        j                  |||      | _        t        j                  ||      | _        t        j                  ||      | _        t        j                  ||      | _        t        j                  ||      | _        t%        |      | _        y )N)eps)dropout)superr   __init__r   	LayerNormnorm1Dropoutr"   dropout1dropout2dropout3MultiheadAttentionmultihead_attnLinearlinear1linear2norm2norm3r   
activation)selfd_modelnheaddim_feedforwardr"   r2   layer_norm_eps	__class__s          r   r$   z'TransformerDecoderLayerOptimal.__init__$   s    ,d<>\\'~>
zz'*

7+

7+

7+ 33GUGT yy/:yy':\\'~>
\\'~>
,Z8r   c                     d|vr't         j                  j                  j                  |d<   t        t
        |   |       y )Nr2   )torchr   
functionalrelur#   r   __setstate__)r3   stater8   s     r   r=   z+TransformerDecoderLayerOptimal.__setstate__8   s5    u$"'(("5"5":":E,,d@Gr   tgtmemorytgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskc           	         || j                  |      z   }| j                  |      }| j                  |||      d   }|| j                  |      z   }| j	                  |      }| j                  | j                  | j                  | j                  |                        }|| j                  |      z   }| j                  |      }|S )Nr   )r(   r&   r,   r)   r0   r/   r"   r2   r.   r*   r1   )r3   r?   r@   rA   rB   rC   rD   tgt2s           r   forwardz&TransformerDecoderLayerOptimal.forward=   s     DMM#&&jjo""37:DMM$''jjo||DLLc9J)KLMDMM$''jjo
r   )      皙?r<   gh㈵>)r   N)NNNN)	__name__
__module____qualname__r$   r=   r   r   rG   __classcell__r8   s   @r   r   r   #   s    W] $9)-9(H
 QU049=<@6 6 Xf=M %f-&.v&6 *2&)9 FLr   r   c                   &     e Zd Zd fd	Zd Z xZS )r   c                 h   t         t        |           |dk  rdn|}||kD  r|}|| _        |dk  rdn|}t	        j
                  ||      | _        d}d}d}t        |||      }	t	        j                  |	|      | _	        t	        j                  ||      | _        | j                  j                  d	       || _        t        ||z  d
z         | _        t         j                  j#                  t!        j$                  ||| j                              | _        t         j                  j#                  t!        j$                  |            | _        t         j                  j*                  j-                  | j&                         t         j                  j*                  j/                  | j(                  d       y )Nr   d      rJ      rI   )r4   r6   r"   )
num_layersFg+?)r#   r   r$   embed_len_decoderr   r-   embed_standartr   TransformerDecoderdecoder	Embeddingquery_embedrequires_grad_r
   intduplicate_factorr:   	Parameterr   duplicate_poolingduplicate_pooling_biasinitxavier_normal_	constant_)r3   r
   num_of_groupsdecoder_embeddingr   rV   decoder_dropoutnum_layers_decoderr6   layer_decoder8   s             r   r$   zMLDecoder.__init__[   sn   i')#01#4C-{* +!2 $5q#8C>O ii(<>OP 5>OFU_np,,\FXY <<(9;LM''. ' #K2C$Ce$K L!&!3!3LL*,=t?T?TU"W&+hh&8&8k9R&S#$$T%;%;< ; ;Q?r   c                    t        |j                        dk(  r"|j                  d      j                  dd      }n|}| j	                  |      }t
        j                  j                  j                  |d      }|j                  d   }| j                  j                  }|j                  d      j                  d|d      }| j                  ||j                  dd            }|j                  dd      }t        j                  |j                  d   |j                  d   | j                  |j                   |j"                        }t%        | j&                        D ]D  }	|d d |	d d f   }
| j(                  |	d d d d f   }t        j*                  |
|      |d d |	d d f<   F |j                  d      d d d | j,                  f   }|| j.                  z  }|}|S )	N      rT   T)inplacer   r   )devicedtype)lenshapeflatten	transposerW   r:   r   r;   r<   r[   weight	unsqueezeexpandrY   zerosr^   rn   ro   rangerV   r`   matmulr
   ra   )r3   xembedding_spatialembedding_spatial_786bsr[   r?   h
out_extrapih_iw_ih_outlogitss                 r   rG   zMLDecoder.forward{   s   qww<1 !		! 6 6q! < ! $ 3 34E F % 3 3 8 89NX\ 8 ]"((+&&--##A&--b"b9LL3==aCDKK1[[QWWQZ9N9NWXW_W_ghgngno
t--. 	9AAq!G*C((Aq1C"',,sC"8Jq!Qw	9 ""1%a):$*:*:):&:;,,,r   )r   rS   rI   )rK   rL   rM   r$   rG   rN   rO   s   @r   r   r   Z   s    @@r   r   )typingr   r:   r   r   torch.nn.modules.transformerr   r   Moduler   r    r   r   <module>r      s:        ;4&RYY &n8		 8r   