gemm.hpp Source File# Composable Kernel: gemm.hpp Source File includeck_tileops tile/ops/gemm.hpp Go to the documentation of this file. 1// SPDX-License-Identifier: MIT 2// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. 3 4#pragma once 5 6#include "ck_tile/ops/gemm/block/block_gemm_areg_bgmem_creg_v1.hpp" 7#include "ck_tile/ops/gemm/block/block_gemm_areg_bgmem_creg_v1_default_policy.hpp" 8#include "ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_v1.hpp" 9#include "ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_v1_custom_policy.hpp" 10#include "ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_v1_default_policy.hpp" 11#include "ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_v2.hpp" 12#include "ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_v2_custom_policy.hpp" 13#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp" 14#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1.hpp" 15#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_custom_policy.hpp" 16#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_default_policy.hpp" 17#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2.hpp" 18#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_custom_policy.hpp" 19#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_default_policy.hpp" 20#include "ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2r1.hpp" 21#include "ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1.hpp" 22#include "ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_custom_policy.hpp" 23#include "ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_default_policy.hpp" 24#include "ck_tile/ops/gemm/block/block_gemm_asmem_bsmem_creg_v1.hpp" 25#include "ck_tile/ops/gemm/block/block_gemm_asmem_bsmem_creg_v1_custom_policy.hpp" 26#include "ck_tile/ops/gemm/block/block_gemm_asmem_bsmem_creg_v1_default_policy.hpp" 27#include "ck_tile/ops/gemm/block/block_gemm_problem.hpp" 28#include "ck_tile/ops/gemm/block/block_universal_gemm_as_bs_cr.hpp" 29#include "ck_tile/ops/gemm/block/block_wp_asmem_bsmem_creg_v1.hpp" 30#include "ck_tile/ops/gemm/block/block_wp_asmem_bsmem_creg_v1_custom_policy.hpp" 31#include "ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp" 32#include "ck_tile/ops/gemm/kernel/gemm_kernel.hpp" 33#include "ck_tile/ops/gemm/kernel/gemm_multi_abd_kernel.hpp" 34#include "ck_tile/ops/gemm/kernel/gemm_multi_d_kernel.hpp" 35#include "ck_tile/ops/gemm/kernel/gemm_tile_partitioner.hpp" 36#include "ck_tile/ops/gemm/kernel/grouped_gemm_kernel.hpp" 37#include "ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp" 38#include "ck_tile/ops/gemm/kernel/streamk_gemm_tile_partitioner.hpp" 39#include "ck_tile/ops/gemm/kernel/streamk_gemm_tile_partitioner_impl.hpp" 40#include "ck_tile/ops/gemm/kernel/universal_gemm_kernel.hpp" 41#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_base.hpp" 42#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp" 43#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_default_policy.hpp" 44#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v3.hpp" 45#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v4.hpp" 46#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v4_default_policy.hpp" 47#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5.hpp" 48#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5_default_policy.hpp" 49#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp" 50#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp" 51#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_mem.hpp" 52#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_scheduler.hpp" 53#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_agmem_bgmem_creg_v1.hpp" 54#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_agmem_bgmem_creg_v1_default_policy.hpp" 55#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_agmem_bgmem_creg_v2.hpp" 56#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_agmem_bgmem_creg_v2_default_policy.hpp" 57#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_problem.hpp" 58#include "ck_tile/ops/gemm/pipeline/gemm_pipelines.hpp" 59#include "ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp" 60#include "ck_tile/ops/gemm/pipeline/tile_gemm_shape.hpp" 61#include "ck_tile/ops/gemm/pipeline/tile_gemm_traits.hpp" 62#include "ck_tile/ops/gemm/pipeline/wp_pipeline_agmem_bgmem_creg_base_policy.hpp" 63#include "ck_tile/ops/gemm/pipeline/wp_pipeline_agmem_bgmem_creg_v2.hpp" 64#include "ck_tile/ops/gemm/warp/warp_gemm.hpp" 65#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma.hpp" 66#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma_impl.hpp" 67#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_smfmac.hpp" 68#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_smfmac_impl.hpp" 69#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_wmma.hpp" 70#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_wmma_impl.hpp" 71#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_wmma_impl_16bit_traits.hpp" 72#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_wmma_impl_8bit_traits.hpp" 73#include "ck_tile/ops/gemm/warp/warp_gemm_attribute_wmma_impl_base_traits.hpp" 74#include "ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp" 75#include "ck_tile/ops/gemm/warp/warp_gemm_impl.hpp" 76#include "ck_tile/ops/gemm/warp/warp_gemm_smfmac_impl.hpp" 77#include "ck_tile/ops/gemm/warp/warp_wmma_gemm.hpp" 78#include "ck_tile/ops/common/generic_2d_block_shape.hpp" 79#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" 80#include "ck_tile/ops/common/streamk_common.hpp" 81#include "ck_tile/ops/common/tensor_layout.hpp" 82#include "ck_tile/ops/common/utils.hpp" batched_gemm_kernel.hpp block_gemm_areg_bgmem_creg_v1.hpp block_gemm_areg_bgmem_creg_v1_default_policy.hpp block_gemm_areg_breg_creg_v1.hpp block_gemm_areg_breg_creg_v1_custom_policy.hpp block_gemm_areg_breg_creg_v1_default_policy.hpp block_gemm_areg_breg_creg_v2.hpp block_gemm_areg_breg_creg_v2_custom_policy.hpp block_gemm_areg_bsmem_creg_one_warp_v1.hpp block_gemm_areg_bsmem_creg_v1.hpp block_gemm_areg_bsmem_creg_v1_custom_policy.hpp block_gemm_areg_bsmem_creg_v1_default_policy.hpp block_gemm_areg_bsmem_creg_v2.hpp block_gemm_areg_bsmem_creg_v2_custom_policy.hpp block_gemm_areg_bsmem_creg_v2_default_policy.hpp block_gemm_areg_bsmem_creg_v2r1.hpp block_gemm_asmem_breg_creg_v1.hpp block_gemm_asmem_breg_creg_v1_custom_policy.hpp block_gemm_asmem_breg_creg_v1_default_policy.hpp block_gemm_asmem_bsmem_creg_v1.hpp block_gemm_asmem_bsmem_creg_v1_custom_policy.hpp block_gemm_asmem_bsmem_creg_v1_default_policy.hpp block_gemm_problem.hpp block_universal_gemm_as_bs_cr.hpp block_wp_asmem_bsmem_creg_v1.hpp block_wp_asmem_bsmem_creg_v1_custom_policy.hpp gemm_kernel.hpp gemm_multi_abd_kernel.hpp gemm_multi_d_kernel.hpp gemm_pipeline_ag_bg_cr_base.hpp gemm_pipeline_ag_bg_cr_comp_async.hpp gemm_pipeline_ag_bg_cr_comp_async_default_policy.hpp gemm_pipeline_ag_bg_cr_comp_v3.hpp gemm_pipeline_ag_bg_cr_comp_v4.hpp gemm_pipeline_ag_bg_cr_comp_v4_default_policy.hpp gemm_pipeline_ag_bg_cr_comp_v5.hpp gemm_pipeline_ag_bg_cr_comp_v5_default_policy.hpp gemm_pipeline_ag_bg_cr_comp_v6.hpp gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp gemm_pipeline_ag_bg_cr_mem.hpp gemm_pipeline_ag_bg_cr_scheduler.hpp gemm_pipeline_agmem_bgmem_creg_v1.hpp gemm_pipeline_agmem_bgmem_creg_v1_default_policy.hpp gemm_pipeline_agmem_bgmem_creg_v2.hpp gemm_pipeline_agmem_bgmem_creg_v2_default_policy.hpp gemm_pipeline_problem.hpp gemm_pipelines.hpp gemm_tile_partitioner.hpp gemm_universal_pipeline_ag_bg_cr_policy.hpp generic_2d_block_shape.hpp grouped_gemm_kernel.hpp load_interleaved_pk_type.hpp streamk_common.hpp streamk_gemm_kernel.hpp streamk_gemm_tile_partitioner.hpp streamk_gemm_tile_partitioner_impl.hpp tensor_layout.hpp tile_gemm_shape.hpp tile_gemm_traits.hpp universal_gemm_kernel.hpp utils.hpp warp_gemm.hpp warp_gemm_attribute_mfma.hpp warp_gemm_attribute_mfma_impl.hpp warp_gemm_attribute_smfmac.hpp warp_gemm_attribute_smfmac_impl.hpp warp_gemm_attribute_wmma.hpp warp_gemm_attribute_wmma_impl.hpp warp_gemm_attribute_wmma_impl_16bit_traits.hpp warp_gemm_attribute_wmma_impl_8bit_traits.hpp warp_gemm_attribute_wmma_impl_base_traits.hpp warp_gemm_dispatcher.hpp warp_gemm_impl.hpp warp_gemm_smfmac_impl.hpp warp_wmma_gemm.hpp wp_pipeline_agmem_bgmem_creg_base_policy.hpp wp_pipeline_agmem_bgmem_creg_v2.hpp