khd8dZddlZddlmZmZmZddlZddlmcm Z ddlmZddl m Z m Z mZmZmZmZddlmZddlmZdd lmZmZmZdd lmZmZmZmZmZm Z dd l!m"Z#gZ$dgd Z%dhdZ&didZ'djdZ(e)ee%dddde%dddde&ddddddde&ddddd dde&dddddddfddddd d!"ee%dd#d$e%ddd$e&dddddd%e&dddd&d d%e&dddd'dd%fddddd d("ee%dd#d$e%ddd$e&dddd'dd%e&dd)dd*d d%e&dd+dd,dd%fddddd d-"ee%dd#d$e%ddd$e&dddd'dd%e&dd)dd*d d%e&dd+dd,dd%fddddd.e)d/0d-1e(de(d2e(d3e(de(d4e(d5e(d6 Z*eGd7d8ejVZ,Gd9d:ejVZ-Gd;dejVZ/ed?e,ed@e/dkdAZ0dkdBZ1dldCZ2eidDe2dEFdGe2dEFdHe2dEFdIe2dEdJKdLe2dEdJKdMe2dEdJKdNe2dEdJKdOe2dEdJKdPe2dEdJKdQe2dEdJKdRe2dEdJKdSe2dEdJKdTe2dEdJKdUe2dEdVdWdXdYe2dEdVdWdXdZe2dEdVdWdXZ3edmd[efd\Z4edmd[efd]Z5edmd[efd^Z6edmd[efd_Z7edmd[efd`Z8edmd[efdaZ9edmd[efdbZ:edmd[efdcZ;edmd[efddZdRdSdTdUdYdZdfy)na  MobileViT Paper: V1: `MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer` - https://arxiv.org/abs/2110.02178 V2: `Separable Self-attention for Mobile Vision Transformers` - https://arxiv.org/abs/2206.02680 MobileVitBlock and checkpoints adapted from https://github.com/apple/ml-cvnets (original copyright below) License: https://github.com/apple/ml-cvnets/blob/main/LICENSE (Apple open source) Rest of code, ByobNet, and Transformer block hacked together by / Copyright 2022, Ross Wightman N)CallableTupleOptional)nn) to_2tuplemake_divisible GroupNorm1ConvMlpDropPath is_exportable)build_model_with_cfg)register_notrace_module)register_modelgenerate_default_cfgsregister_model_deprecations)register_block ByoBlockCfg ByoModelCfgByobNetLayerFn num_groups)Blockc <td|||d|tddS)Nbottler T) bottle_in linear_out)typedcsgsbr block_kwargs)rdictrr r!r#s Q/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/models/mobilevit.py_inverted_residual_blockr(!s&  a1rDT: <<c Xt||||tdd|dt|||fS)Nr& mobilevitr )transformer_dimtransformer_depth patch_size)rrr r!r$r(rr%rr r!r-r.r/r#s r'_mobilevit_blockr2(s> !1Q26Q! /"3%'   r)@?c Zt||||tdd|d|dt||fS)Nr& mobilevit2r )r.r/)rrr r!r#r"r$r0)rr r!r.r/r#transformer_brs r'_mobilevitv2_blockr96s? !1Q26a1A"3%'  r)?c Ld}|dk7r%t|Dcgc]}t||zc}}ttd|dddtd|dddt d|dddt d|d dd t d|d dd ftd |zd d d d}|Scc}w)N)@ir:r rr4r&r3)rr r!r.r* 3x3silu)blocksstem_chs stem_type stem_pool downsample act_layer)tupleintrr(r9) multiplierchsr cfgs r'_mobilevitv2_cfgrPCs "CS#6QSZ(67  $qCFaC @ $qCFaC @ c!fQ G c!fQ G c!fQ G  R*_% C J7sB!r&r@0r<r1P`rBrCrDi@)rErFrGrHrIrJ num_featuresrA)rr r!)rr r!r-r.r/xr?r=iseg?)rd_ratio)rErFrGrHrI attn_layer attn_kwargsrVg?g?g?g?) mobilevit_xxs mobilevit_xs mobilevit_s semobilevit_smobilevitv2_050mobilevitv2_075mobilevitv2_125mobilevitv2_100mobilevitv2_150mobilevitv2_175mobilevitv2_200c%eZdZdZdddddddddd d d d d d dej fd edeededededeede eefdedeedededededede dede de f$fd Z d ejd!ejfd"ZxZS)#MobileVitBlockzS MobileViT block Paper: https://arxiv.org/abs/2110.02178?context=cs.LG Nr@r r:r r r4r3r*Fin_chsout_chs kernel_sizestride bottle_ratio group_sizedilation mlp_ratior-r.r/ num_heads attn_dropdrop no_fusiondrop_path_ratelayerstransformer_norm_layerc tt| |xs t}t ||}|xs|}| xst ||z} |j ||||||d|_tj|| dd|_ tjt| Dcgc] }t| || d| |||j| "c}|_|| |_|j | |dd|_|rd|_n|j ||z||d|_t'| |_|j(d|j(dz|_ycc}w) Nrrrrsgroupsrvr FrrbiasT)rwrxqkv_biasry proj_drop drop_pathrJ norm_layerrrrs)superrl__init__rrr conv_norm_actconv_kxkrConv2dconv_1x1 SequentialrangeTransformerBlockact transformernorm conv_proj conv_fusionrr/ patch_area)selfrprqrrrsrtrurvrwr-r.r/rxryrzr{r|r}r~kwargsr_ __class__s r'rzMobileVitBlock.__init__sg, nd,.$79J/#V)R^L6GG9 FM>&'ggbcl# m \ ) X W X ) XX ==''(;!);<##$7T^^a%78 XXq$..1qX9 s ==''(:;;?==;M;M;Y##DNNQ$67_c 5b17.33D3IffUmn66u==mmC mmC  r)cN||j|S|j||S)N)r)rr)rrrs r'rzLinearSelfAttention.forwards. >**1- -++Af+= =r))roroTN)rrrrrLrrrrrrjitignorerrrrrs@r'rrs, ... .  .  .2ELLU\\0 YY(U\\(8ELL;Q(]b]i]i((T>>x /E>QVQ]Q]>r)rc eZdZdZ d dedededededdf fd Zdd ejd e ejdejfd Z xZ S)LinearTransformerBlockaF This class defines the pre-norm transformer encoder with linear self-attention in `MobileViTv2 paper <>`_ Args: embed_dim (int): :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, P, N)` mlp_ratio (float): Inner dimension ratio of the FFN relative to embed_dim drop (float): Dropout rate. Default: 0.0 attn_drop (float): Dropout rate for attention in multi-head attention. Default: 0.0 drop_path (float): Stochastic depth rate Default: 0.0 norm_layer (Callable): Normalization layer. Default: layer_norm_2d Shape: - Input: :math:`(B, C_{in}, P, N)` where :math:`B` is batch size, :math:`C_{in}` is input embedding dim, :math:`P` is number of pixels in a patch, and :math:`N` is number of patches, - Output: same shape as the input Nrrwrzryrrc8t||xstj}|xst}|||_t ||||_t||_ |||_ t|t||z|||_ t||_y)N)rryr) in_featureshidden_featuresrJrz)rrrSiLUr norm1rattnr drop_path1norm2r rLmlp drop_path2) rrrwrzryrrJrrs r'rzLinearTransformerBlock.__init__s ( -:  * ')y\`a "9- * ! I 56  #9-r)rrcF|3||j|j|j|z}n9|}|j|}|j||}|j||z}||j|j |j |z}|Sr)rrrrrr)rrrress r'rzLinearTransformerBlock.forwards >DOODIIdjjm$<==AC 1 A !V$A"S(A A 78 8r))r4rororoNNr) rrrrrLrrrrrrrrs@r'rrs $... .  .  . .4  x /E QVQ]Q] r)rceZdZdZddddddddd d d d defd ed eed ededeedeeefdedeedededededede de ffd Z de jde jfdZxZS)MobileVitV2Blockz8 This class defines the `MobileViTv2 block <>`_ Nr@r:r rmr4r3rnrorprqrrrtrurvrwr-r.r/ryrzr|r}r~c xtt| |xs t}t ||}|xs|}|xst ||z}|j |||d||d|_tj||dd|_ tjt| Dcgc]}t||| | | |j| c}|_|||_|j ||ddd|_t%| |_|j&d|j&dz|_t+|_ycc}w)Nr rrFr)rwryrzrrJr)rrrs apply_act)rrrrrrrrrrrrrrrrrrrr/rr coreml_exportable)rrprqrrrtrurvrwr-r.r/ryrzr|r}r~rrrrs r'rzMobileVitV2Block.__init__s@& .0$79J/#V)R^L6 * -8*W%j>UD1    r)c dtt||f|s t|nt|tdd|Srrrs r'_create_mobilevit2r+rr)c "|ddddddddd d d |S) Ni)r@r>r>)rnrng?bicubic)rororo)r:r:r:z stem.convzhead.fcF) url num_classes input_size pool_sizecrop_pct interpolationmeanstd first_conv classifierfixed_input_size)rrs r'_cfgr*3s24}SY)\!!     r)zmobilevit_xxs.cvnets_in1kztimm/) hf_hub_idzmobilevit_xs.cvnets_in1kzmobilevit_s.cvnets_in1kzmobilevitv2_050.cvnets_in1kg"~j?)r+r"zmobilevitv2_075.cvnets_in1kzmobilevitv2_100.cvnets_in1kzmobilevitv2_125.cvnets_in1kzmobilevitv2_150.cvnets_in1kzmobilevitv2_175.cvnets_in1kzmobilevitv2_200.cvnets_in1kz$mobilevitv2_150.cvnets_in22k_ft_in1kz$mobilevitv2_175.cvnets_in22k_ft_in1kz$mobilevitv2_200.cvnets_in22k_ft_in1kz(mobilevitv2_150.cvnets_in22k_ft_in1k_384)r@r?r?) r,)r+r r!r"z(mobilevitv2_175.cvnets_in22k_ft_in1k_384z(mobilevitv2_200.cvnets_in22k_ft_in1k_384rc tdd|i|S)Nr)r`rrrs r'r`r`os  N Nv NNr)c tdd|i|S)Nr)rar.r/s r'rarats  M Mf MMr)c tdd|i|S)Nr)rbr.r/s r'rbrbys  Lz LV LLr)c tdd|i|S)Nr)rdr.r/s r'rdrd~  P: P PPr)c tdd|i|S)Nr)rer.r/s r'rerer3r)c tdd|i|S)Nr)rgr.r/s r'rgrgr3r)c tdd|i|S)Nr)rfr.r/s r'rfrfr3r)c tdd|i|S)Nr)rhr.r/s r'rhrhr3r)c tdd|i|S)Nr)rir.r/s r'ririr3r)c tdd|i|S)Nr)rjr.r/s r'rjrjr3r))mobilevitv2_150_in22ft1kmobilevitv2_175_in22ft1kmobilevitv2_200_in22ft1kmobilevitv2_150_384_in22ft1kmobilevitv2_175_384_in22ft1kmobilevitv2_200_384_in22ft1k)@)r*r@)r3r4r5)r:)NF)rC)F)?rrtypingrrrrtorch.nn.functionalr functionalr timm.layersrrr r r r _builderr _features_fxr _registryrrrbyobnetrrrrrrvision_transformerrr__all__r(r2r9rPr%rModulerlrrrrrr* default_cfgsr`rarbrdrergrfrhrirjrr)r)r'rMs  ,, __*1YY[[9 <  * $qB! < $qB! < qB!RSTabgj k qB!RSTabgj k qB!RSTabgj k    $qB! 4 $qB! 4 qB!RSTab c qB!STUbc d qB!STUbc d    $qB! 4 $qB! 4 qB!STUbc d qC1cUVcd e qC1cUVcd e    $qB! 4 $qB! 4 qB!STUbc d qC1cUVcd e qC1cUVcd e  #&"%S)$S)$T*$S)$S)$T*$S)QI XhRYYhhVt>"))t>n7RYY7tZryyZZz{N+|-.%.&!8.&w 7.&tg6.& "4$ .&"4$.&"4$.&"4$.&""4$#.&("4$).&."4$/.&6+D-7.&<+D-=.&B+D-C.&J/ Hs1DK.&P/ Hs1DQ.&V/ Hs1DW.&. bOOONNNMwMMQ7QQQ7QQQ7QQQ7QQQ7QQQ7QQQ7QQH F F F$N$N$N'r)