o h@sddlZddlmZddlmZddlmZddlmZm Z m Z ddl Z ddl Z ddl mZddlmZmZddlmZdd lmZd d lmZmZmZd d lmZd d lmZmZgdZeGdddZ dee!de!fddZ"de j#de!de!de$e j#e!ffddZ%de j#de!de!de!de j#f ddZ&e j'(de j'(dGdddej)Z*de j#d e!de j#fd!d"Z+d#e j#d$e j#d%e$e!e!e!fd&e$e!e!e!fd'e j#d(e j#d)e j#de j#fd*d+Z,de j#d,e j#d-e-fd.d/Z.e j'(d+e j'(d/Gd0d1d1ej)Z/Gd2d3d3ej)Z0Gd4d5d5ej)Z1Gd6d7d7ej)Z2d8e3e d9e4d:e ed;e-dZ5Gd?d@d@eZ6GdAdBdBeZ7eedCe6j8fdDddEdFd:e e6d;e-dr?r@layers __class__r'r(rBBs    z Pool.__init__r0thwc Cst|dd\}}tj|ddd\}}|dd}|jdd\}}}||||f|}|jr<|jdur<||}| |}|jdd\}} } ||||ddd}tj ||fdd}|jsm|jdurm||}t |dd|}||| | ffS) Nr,r,r )indicesr3rr3) r9torchZ tensor_split transposer6reshape contiguousr@rFr=catr;) rGr0rKr8 class_tokenBNCTHWr'r'r(forwardSs    z Pool.forward)NF)r!r"r#rDModulerboolrBrQTensortupler$r] __classcell__r'r'rIr(r<As>r< embeddingdcCs@|jd|kr |Stjj|ddd|dddddS)Nrr,Zlinear)sizemode)r6rDZ functionalZ interpolatepermuter4r:)rcrdr'r'r( _interpolatemsrhattnqq_thwk_thw rel_pos_h rel_pos_w rel_pos_tc% Cs6|\}}} |\} } } tdt|| d} tdt| | d}tdt|| d}t| |d}t|| d}t|dddf|t| dddfd| |}t| | d}t| | d}t| dddf|t| dddfd| |}t| |d}t|| d}t|dddf|t| dddfd| |}t|| }t||}t||}||}||}||}|j\}}}}|ddddddf||||| |} td| |}!td| |}"| dddddd ||||| |} t | | dd dd}#|# |||| ||  dddddd }#|!ddddddddddddddf|"ddddddddddddddf|#ddddddddddddddf||||| | | | }$|ddddddddf|$7<|S) Nr r,?zbythwc,hkc->bythwkzbythwc,wkc->bythwkrrrL) r$maxrQZarangerhlongr6rSZeinsumrgmatmulrRview)%rirjrkrlrmrnroZq_tZq_hZq_wZk_tZk_hZk_wZdhZdwdtZ q_h_ratioZ k_h_ratioZdist_hZ q_w_ratioZ k_w_ratioZdist_wZ q_t_ratioZ k_t_ratioZdist_tZRhZRwZRtrWZn_head_r3Zr_qZrel_h_qZrel_w_qZrel_q_tZrel_posr'r'r( _add_rel_pos|sH <<<      **$...(rxshortcutresidual_with_cls_embedc CsZ|r |||S|ddddddddf|ddddddddf7<|Sr+)add_)r0ryrzr'r'r( _add_shortcuts  Dr|cseZdZdejfdeededededeedeedeed eed ed ed ed ede dej fddffdd Z de j deeeefdee j eeeefffddZZS)MultiscaleAttention input_size embed_dim output_dimrrrrr residual_poolrz rel_pos_embeddropout norm_layer.r*Nc spt||_||_||_|||_dt|j|_| |_ | |_ t |d||_ t ||g}| dkr@|t j| ddt j||_d|_t|dksUt|dkrrdd|D}tt j|j|j||||jd d | |j|_d|_d|_t|dkst|dkrd d|D}tt j|j|j||||jd d | |j|_tt j|j|j||||jd d | |j|_d|_d|_d|_| r6t|dd}t|d kr||dn|}t|d kr||dn|}d t||d}d |d d}t t||j|_t t||j|_t t||j|_t j j!|jddt j j!|jddt j j!|jdddSdS)Nrprr~Tinplacer,cSg|]}t|dqSr r$).0rjr'r'r( z0MultiscaleAttention.__init__..F)stridepaddinggroupsbiascSrrr)rkvr'r'r(rrrr {Gz?std)"rArBrrrhead_dimmathsqrtscalerrrzrDLinearqkvrCDropoutrEprojectpool_qr/r<Conv3dpool_kpool_vrmrnrorrlen ParameterrQzerosinit trunc_normal_)rGrrrrrrrr rrzrrrrHZ padding_qZ padding_kvreZq_sizeZkv_sizeZ spatial_dimZ temporal_dimrIr'r(rBs         zMultiscaleAttention.__init__r0rKc Cs:|j\}}}||||d|j|jddjdd\}}}|jdur-|||\}} n|} |jdur<|||d}|j durI| ||\}}t |j ||dd} |j durr|jdurr|jdurrt| ||| |j |j|j} | jdd} t | |}|jrt|||j|dd|d|j}||}||fS)Nrr,r rPrrO)r6rrSrrrRZunbindrrrrQrtrrmrnrorxZsoftmaxrr|rzrr) rGr0rKrWrXrYrjkr.rlrir'r'r(r]!s6 2     zMultiscaleAttention.forward)r!r"r#rD LayerNormr&r$r_floatrr^rBrQr`rar]rbr'r'rIr(r}sB      >\r}cseZdZddejfdeedededededede d e d e d ej fd d ffdd Z de jdeeeefd ee jeeeefffddZZS)MultiscaleBlockr~rcnfrrzrproj_after_attnrstochastic_depth_probr.r*Nc st||_d|_t|jdkr.dd|jD} dd| D} ttj| |j| dd|_|r3|j n|j } | |j |_ | | |_ t |j tj|_t||j | |j|j|j|j|j||||| d |_t| d| |j gtj|dd|_t|d |_d|_|j |j krt|j |j |_dSdS) Nr,cSs g|] }|dkr |dn|qSrMr')rr)r'r'r(rVs z,MultiscaleBlock.__init__..cSrrr)rrr'r'r(rWr)rr) rrrr rrrzrrrL)Zactivation_layerrrrow)rArBr pool_skipr/rr<rDZ MaxPool3drrnorm1norm2 isinstanceZ BatchNorm1dneeds_transposalr}rrrr rir ZGELUmlpr stochastic_depthrr) rGrrrrzrrrrrZ kernel_skipZ padding_skipZattn_dimrIr'r(rBEsP      zMultiscaleBlock.__init__r0rKc Cs|jr||ddddn||}|||\}}|jdus%|js'|n||}|jdur3|n|||d}|||}|jrR||ddddn||}|jdus_|jra|n||}||| ||fS)Nr,r r) rrrRrirrrrrr) rGr0rKZx_norm1Zx_attnZthw_newZx_skipZx_norm2Zx_projr'r'r(r]s**zMultiscaleBlock.forward)r!r"r#rDrr&r$rr_rrr^rBrQr`rar]rbr'r'rIr(rDs4    >:rc sPeZdZdedeeefdededdf fdd Zd ejdejfd d Z Z S) PositionalEncoding embed_size spatial_size temporal_sizerr*Ncst||_||_tt||_d|_ d|_ d|_ |sGtt|jd|jd||_ tt|j||_ tt||_ dSdS)Nrr,) rArBrrrDrrQrrV spatial_pos temporal_pos class_pos)rGrrrrrIr'r(rBs $zPositionalEncoding.__init__r0cCs|j|ddd}tj||fdd}|jdur\|jdur\|jdur\|jj \}}tj |j|dd}| |jd|j dd d|tj|jd|fddd}| ||S)NrrOr,rP)rVexpandrer4rQrUrrrr6Zrepeat_interleaver{rrS)rGr0rVZhw_sizerZ pos_embeddingr'r'r(r]s &  zPositionalEncoding.forward) r!r"r#r$rar_rBrQr`r]rbr'r'rIr(rs*rc$seZdZ         ddeeefd ed eed ed ed edededededede e de j fde e de j fdeeeefdeeeefdeeeefddf"fdd Z dejdejfddZZS) r?r~Nrrr rLrLr,rrrr block_settingrrzrrrattention_dropoutr num_classesblock.rpatch_embed_kernelpatch_embed_stridepatch_embed_paddingr*cstt|t|}|dkrtd| durt} | dur&ttjdd} tj d|dj |||d|_ dd t |f||j j D}t|dj |d |d f|d|d |_t|_t|D]/\}}| ||d }|j| ||||||| || d t|jdkrdd t ||jD}q`| |dj|_ttj|ddt|dj| |_|D][}t|tjrtjj|jddt|tjr|j durtj!|j dqt|tjr|jdurtj!|jd |j durtj!|j dqt|tr |"D] }tjj|ddqqdS)a MViT main class. Args: spatial_size (tuple of ints): The spacial size of the input as ``(H, W)``. temporal_size (int): The temporal size ``T`` of the input. block_setting (sequence of MSBlockConfig): The Network structure. residual_pool (bool): If True, use MViTv2 pooling residual connection. residual_with_cls_embed (bool): If True, the addition on the residual connection will include the class embedding. rel_pos_embed (bool): If True, use MViTv2's relative positional embeddings. proj_after_attn (bool): If True, apply the projection after the attention. dropout (float): Dropout rate. Default: 0.0. attention_dropout (float): Attention dropout rate. Default: 0.0. stochastic_depth_prob: (float): Stochastic depth rate. Default: 0.0. num_classes (int): The number of classes. block (callable, optional): Module specifying the layer which consists of the attention and mlp. norm_layer (callable, optional): Module specifying the normalization layer to use. patch_embed_kernel (tuple of ints): The kernel of the convolution that patchifies the input. patch_embed_stride (tuple of ints): The stride of the convolution that patchifies the input. patch_embed_padding (tuple of ints): The padding of the convolution that patchifies the input. rz+The configuration parameter can't be empty.Ngư>)epsr)Z in_channelsZ out_channelsZ kernel_sizerrcSg|]\}}||qSr'r'rrerr'r'r(rrz!MViT.__init__..r,r )rrrrrp) rrrrzrrrrrcSrr'r'rr'r'r(r rrOTrrrr~)#rArBr rr5rrrDrrr conv_projziprr pos_encodingZ ModuleListblocks enumeraterCrrr>rErrheadmodulesrrrweightrZ constant_ parameters)rGrrrrrzrrrrrrrrrrrZtotal_stage_blocksrZstage_block_idrZsd_probmweightsrIr'r(rBs )         z MViT.__init__r0cCst|ddd}||}|ddd}||}|jjf|jj}|jD] }|||\}}q'||}|dddf}| |}|S)Nrqr rr,) r9rflattenrRrrrrr>r)rGr0rKrr'r'r(r]"s     z MViT.forward) rr~r~rNNrrr)r!r"r#rar$rrr_rrrrDr^rBrQr`r]rbr'r'rIr(rs\          xrrrrprogresskwargsc Ks|dur1t|dt|jd|jdd|jddksJt|d|jdt|d|jd|dd }|dd }td||||d d |d d|dd |dd |d|}|durk||j|dd|S)Nr categoriesmin_sizerr,rrmin_temporal_sizerrFrzTrr)rrrrrzrrr)rZ check_hashr')rrmetapoprZload_state_dictZget_state_dict)rrrrrrrmodelr'r'r(_mvit9s,         rc @JeZdZedeedddddddedd d d d d diddd dZeZdS)rz:https://download.pytorch.org/models/mvit_v1_b-dbeb1030.pthr?rr?rrZ crop_sizeZ resize_sizemeanrrzShttps://github.com/facebookresearch/pytorchvideo/blob/main/docs/source/model_zoo.mdThe weights were ported from the paper. The accuracies are estimated on video-level with parameters `frame_rate=7.5`, `clips_per_video=5`, and `clip_len=16`ip. Kinetics-400gJ +S@gh|?eW@zacc@1zacc@5guVQ@g rxa@ rrrZrecipeZ_docsZ num_paramsZ_metricsZ_ops _file_sizeurlZ transformsrN r!r"r#rrr rKINETICS400_V1DEFAULTr'r'r'r(rZ4rc @r)rz:https://download.pytorch.org/models/mvit_v2_s-ae3be167.pthrrrrrrzChttps://github.com/facebookresearch/SlowFast/blob/main/MODEL_ZOO.mdrirg r0T@g(\W@rguVP@g?5^I|`@rrNrr'r'r'r(r{rrZ pretrained)rT)rrcKst|}gdgdgdggdggdgggggggggggdgggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgggdggdgggggggggggdgggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgd }g}tt|d D],}|t|d ||d ||d ||d ||d||d||d|d qtddd|dd|dd||d|S)a Constructs a base MViTV1 architecture from `Multiscale Vision Transformers `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V1_B_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V1_B_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V1_B_Weights :members: r,r r rLrLrLrLrLrLrLrLrLrLrLr`rrrrrrrrrrrr)rrrrrrrrrrrrrrrrrrrr,r r r,rrr,rLrLr,r,r,rrrrrrr rrrrrrr rrFr皙?)rrrrrzrrrNr')rverifyrangerrCrrrrrrconfigrir'r'r(rs ..,          rcKsDt|}gdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgd }g}tt|d D],}|t|d ||d ||d ||d ||d||d||d|d qtddd|dddd|dd||d |S)aCConstructs a small MViTV2 architecture from `Multiscale Vision Transformers `__ and `MViTv2: Improved Multiscale Vision Transformers for Classification and Detection `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V2_S_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V2_S_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V2_S_Weights :members: r)rrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrTFrr) rrrrrzrrrrrNr')rrr rrCrrrr r'r'r(rs N          r);rcollections.abcr dataclassesr functoolsrtypingrrrrQZtorch.fxZtorch.nnrDopsr r Ztransforms._presetsr utilsr Z_apirrr_metarZ_utilsrr__all__rr$r/r`rar9r;Zfxwrapr^r<rhrxr_r|r}rrrr&rrrrrrrr'r'r'r(s         &"  ,   <  H !!!*`.