kh=dZddlZddlmZmZmZmZddlZddlmZddl m Z ddl m Z  d3d e d ed ed eej fd Z d4d e dede d eej dej$f dZddddej(dfdee de dedededej*d eej dej$fdZddddddddddej(df dee deej$d e de ded ed ed!ed"eee d#ed$edej*d eej deej$fd%ZGd&d'ej2Zd(Zd)ej$fd*Zd)eej$fd+Zd)ej$fd,Zd-Zdddddddddej(df dee deej$de de ded ed!ed"eee d#ed$edej*d eej fd.Z Gd/d0ej2Z!Gd1d2ej2Z"y)5zv Sin-cos, fourier, rotary position embedding modules and functions Hacked together by / Copyright 2022 Ross Wightman N)ListTupleOptionalUnion)nn)ndgrid)_assertT num_bandsmax_freq linear_bandsdevicec|r,tjd|dz |tj|}nBdtjdtj|ddz |tj|z}|tj zS)N?dtyperrr)torchlinspacefloat32mathlogpi)r r r rbandss X/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/pos_embed_sincos.pypixel_freq_bandsrsh sHqL)5==Y_`U^^Atxx!'>"  2 2D(QIkk599T?EIIdO<)LTTUVWG ::E: ""sA E ijrmax_res include_grid in_pixelsref_feat_shape grid_offset grid_indexingc @|+|rt|t||| }n,t||d| }n| |j} | |j} |r6|D cgc]*} t j dd| | t j,}} nS|D cgc]H} t j| | t jjt j| zJ}} |(t|||Dcgc]\}}}||z |z}}}}t jt j|| d }|jd }||z}|jj| |j!j| }}|r|||g}|S||g}|Scc} wcc} wcc}}}w) a Args: feat_shape: Feature shape for embedding. bands: Pre-calculated frequency bands. num_bands: Number of frequency bands (determines output dim). max_res: Maximum resolution for pixel based freq. temperature: Temperature for non-pixel freq. linear_bands: Linear band spacing for pixel based freq. include_grid: Include the spatial grid in output. in_pixels: Output in pixel freq. ref_feat_shape: Reference feature shape for resize / fine-tune. grid_offset: Constant offset to add to grid for non-pixel freq. grid_indexing: Indexing mode for meshgrid ('ij' or 'xy') dtype: Output dtype. device: Output device. Returns: )r rrr/gr)stepsrrr1)indexingr0r2r3)rfloatr'rrrrrr#r$r%zipr4meshgridr7r8r9)r)rr rDrr rErFrGrHrIrrr;txfrr<pospos_sinpos_cosouts rbuild_fourier_pos_embedrXPsF } $g) E' E >\\F =KKE   NN3!F%-- P     LL6 = @ @ OR] ]  !&)!Z&H I I71aQUQY I I ;;u~~a-@b ID >>" D ,Cwwy||%|0#''),,u2EWG&24' "C J:A'8JC J)   Js/FA F(Fc8eZdZ ddedeffd ZdZxZS) FourierEmbedrDr ct|||_||_||_||_|j dt||dy)NrF persistent)super__init__rDr concat_grid keep_spatialregister_bufferr)selfrDr r`ra __class__s rr_zFourierEmbed.__init__sQ  "&(   Wi 0  rc |jdd\}}|jdd}t||j|j|j|j }t j|d}|jddjt|}|fd|jdz zz}|jrKt j||jdj|jdd ddgd}|St j|jddd d|jdj|gd}|j!||j#d}|S) Nr)rErrr0r2)r0rr)shaperXrr`rrrcatr6r5lenndimrar7expandpermutereshapenumel)rcrQBCr)emb batch_expands rforwardzFourierEmbed.forwardsCwwr{1WWQR[ %  JJ))''88  ii$mmB#++C O<teqvvz22     1cmmA.55lCKKAqRSUVWX^_`A  199Q1a0#--2B2I2I,2WX^`aA !Z--/4Ar)rAr(TF)__name__ __module__ __qualname__intr_rt __classcell__rds@rrZrZs-    $rrZctj|ddddf |ddddfgdj|jS)N.rrr0)rr4rnrh)rQs rrotr|sC ;;319 qcc{3R 8 @ @ IIrrQc|jdk(rP||jdj|zt||jdj|zzS||zt||zzS)Nrgr)rkr7 expand_asr|)rQsin_embcos_embs rapply_rot_embedrsm||q7$$Q'11!44s1v@Q@QRS@T@^@^_`@a7aaa w;Q') ))rct|tjr|g}|Dcgc]}||zt||zzc}Scc}wN) isinstancerTensorr|)rQrrrPs rapply_rot_embed_listrs=!U\\" C45 6qAK#a&7* * 66 6sAc|jdd\}}|jdk(rP||jdj|zt ||jdj|zzS||zt ||zzS)Nrr0rgr) tensor_splitrkr7r~r|)rQrrrrs rapply_rot_embed_catrs''2.GW||q7$$Q'11!44s1v@Q@QRS@T@^@^_`@a7aaa w;Q') ))rc |jdj|jddd}|jd|jdjdd|jd}|S)Nrr0r)r7rlrhgather)rQ pos_embed keep_indicess rapply_keep_indices_nlcrse##A&--aggaj"bAI  L$:$:2$>$E$Eb"ioo^`Na$bcI rc t|||dz||||||| | |  \} } d}|D]}||z} | j|djdd} | j|djdd} | | fS)a Args: feat_shape: Spatial shape of the target tensor for embedding. bands: Optional pre-generated frequency bands dim: Output dimension of embedding tensor. max_res: Maximum resolution for pixel mode. temperature: Temperature (inv freq) for non-pixel mode linear_bands: Linearly (instead of log) spaced bands for pixel mode in_pixels: Pixel vs language (inv freq) mode. ref_feat_shape: Reference feature shape for resize / fine-tune. grid_offset: Constant offset to add to grid for non-pixel freq. grid_indexing: Indexing mode for meshgrid ('ij' or 'xy') dtype: Output dtype. device: Output device. Returns: r.) rr rDrr rFrGrHrIrrrr0r)rXrnrepeat_interleave)r)rr*rDrr rFrGrHrIrrrrnum_spatial_dimrQs rbuild_rotary_pos_embedrsB/(!%# GWO 1ooor2DDQKGooor2DDQKG G rc eZdZdZ d dedeeedeeedede f fd Z d deeefd Z d Z xZ S) RotaryEmbeddinga Rotary position embedding NOTE: This is my initial attempt at impl rotary embedding for spatial use, it has not been well tested, and will likely change. It will be moved to its own file. The following impl/resources were referenced for this impl: * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py * https://blog.eleuther.ai/rotary-embeddings/ r r)rGrHrIc t |||_||_||_||_||_||_||_| |_ |Q|rt|dzt||} nt|dz|d} |jd| dd|_d|_yt!||||||j|j|j\} } d|_|jd | d|jd | dy) Nr.r rrr rFr\r)r*rDr rFrGrHrI pos_embed_sin pos_embed_cos)r^r_r*rDrrFr)rGrHrIrrMr'rbrrrr)rcr*rDrrFr r)rGrHrIremb_sinemb_cosrds rr_zRotaryEmbedding.__init__,s7  &"$,&*  (1H'N!- #1H +    !  "&D !%D  6%)##22 ,,"00  GWDJ    !     ! rrhc|jG|Jt||j|j|j|j|j S|j |jfS)NrFrGrHrI)rrrFrGrHrIrr)rcrhs r get_embedzRotaryEmbedding.get_embedosj :: !$ $$) ..#22 ,,"00  %%t'9'99 9rc^|j|jdd\}}t|||SNr)rrhr)rcrQrrs rrtzRotaryEmbedding.forward~s.>>!''!"+6q'733rrAi'TFNNrBrCrrurvrw__doc__boolrrrxrMstrr_rrtryrzs@rrr!s!&.226!#!%A  A!c+A%T#Y/AAAF :xS 2 :4rrc eZdZdZ d dedeeedeeedede f fd Z d deeefd Z d Z xZ S) RotaryEmbeddingCata Rotary position embedding w/ concatenatd sin & cos The following impl/resources were referenced for this impl: * https://github.com/lucidrains/vit-pytorch/blob/6f3a5fcf0bca1c5ec33a35ef48d97213709df4ba/vit_pytorch/rvt.py * https://blog.eleuther.ai/rotary-embeddings/ r r)rGrHrIc t |||_||_||_||_||_||_||_| |_ |J|rt|dzt||} nt|dz|d} |jd| dd|_yt||||||j|j|j} d|_|jd t#j$| d dy) Nr.rrrrFr\rrr0)r^r_r*rDrrFr)rGrHrIrrMr'rbrrrrri) rcr*rDrrFr r)rGrHrIrembedsrds rr_zRotaryEmbeddingCat.__init__s  &"$,&*  (1H'N!- #1H +    !  "DN,%)##22 ,,"00 FDJ   &"%  ! rrhc|j[|Yt||j|j|j|j|j }t j|dS|j |jSJd)Nrr0zPget_embed() requires pre-computed pos_embed or valid shape w/ pre-computed bands) rrrFrGrHrIrrir)rcrhrs rrzRotaryEmbeddingCat.get_embedsy :: !e&7+ ..#22 ,,"00 F99VR( ( ^^ '>> ! ll l5rcV|j|jdd}t||Sr)rrhr)rcrQrs rrtzRotaryEmbeddingCat.forwards'NN17712;/ "1i00rrrrrzs@rrrs!&.226!#!%;  ;!c+;%T#Y/;;;zmxS 2m"1rr)gl@TN)rrN)#rrtypingrrrrrrr<r trace_utilsr rxrMrrrrr'rrr@rrXModulerZr|rrrrrrrrrrsU //  !)-    & $)- &   \\ ###("]])-&#I&# &#&# &# ! &# {{ &#&&# \\&#V)-#"".2!"]])-MIM %MM M  M  MMM!c+MMM{{M&M %,,M`)299)XJ*u||* 7D.7 *5<<*)-#".2!"]])-5I5 %55 5  5  55!c+555{{5&5p`4bii`4FY1Y1r