{ "metadata": { "timestamp": "2026-09-05T13:01:03.947452+00:00", "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", "torch_version": "2.14.0", "torch_git_version": "08187d9e0fba026dc8217405802ab5381dc88d90", "torch_config": "PyTorch built with:\n - GCC 4.2\n - C++ Version: 202002\n - clang 21.0.0\n - OpenMP 202011\n - LAPACK is enabled (usually provided by MKL)\n - NNPACK is enabled\n - CPU capability usage: DEFAULT\n - Build settings: BLAS_INFO=accelerate, BUILD_TYPE=Release, COMMIT_SHA=08187d9e0fba026dc8217405802ab5381dc88d90, CUDA_FLAGS= -DLIBCUDACXX_ENABLE_SIMPLIFIED_COMPLEX_OPERATIONS -Xfatbin -compress-all -Wno-deprecated-gpu-targets --expt-extended-lambda -DCUB_WRAPPED_NAMESPACE=at_cuda_detail -DDISABLE_CUSPARSE_DEPRECATED -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -DC10_NODEPRECATED, CXX_COMPILER=/usr/bin/c++, CXX_FLAGS= -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DUSE_PYTORCH_QNNPACK -DAT_BUILD_ARM_VEC256_WITH_SLEEF -DUSE_PYTORCH_METAL_EXPORT -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -DUSE_COREML_DELEGATE -O2 -fPIC -DC10_NODEPRECATED -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=braced-scalar-init -Werror=range-loop-construct -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-unknown-pragmas -Wno-unused-parameter -Wno-strict-overflow -Wno-strict-aliasing -Wvla-extension -Wsuggest-override -Wnewline-eof -Winconsistent-missing-override -Winconsistent-missing-destructor-override -Wno-pass-failed -Wno-error=old-style-cast -Wconstant-conversion -Qunused-arguments -faligned-new -fno-math-errno -fno-trapping-math -Werror=format -fclang-abi-compat=17 -DUSE_MPS -Wno-missing-braces, LAPACK_INFO=accelerate, TORCH_VERSION=2.14.0, USE_CUDA=OFF, USE_CUDNN=OFF, USE_CUSPARSELT=OFF, USE_EIGEN_FOR_BLAS=ON, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_GLOO=ON, USE_HIPSPARSELT=OFF, USE_MKL=OFF, USE_MKLDNN=OFF, USE_MPI=OFF, USE_NCCL=OFF, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, USE_ROCM_KERNEL_ASSERT=OFF, USE_XCCL=OFF, USE_XPU=OFF, \n", "threads": 8, "thread_environment": { "TVM_NUM_THREADS": "8", "OMP_NUM_THREADS": "8", "VECLIB_MAXIMUM_THREADS": "8" }, "git_revision": "86dbcb072d0a4a166c91aeb237f943a39e9d0f17", "worktree_dirty": true, "removed_environment": { "PYTORCH_MPS_FAST_MATH": null, "PYTORCH_MPS_PREFER_METAL": null, "PYTORCH_ENABLE_MPS_FALLBACK": null, "LUISA_ENABLE_VALIDATION": null, "MTL_DEBUG_LAYER": null, "MTL_SHADER_VALIDATION": null, "DYLD_PRINT_LIBRARIES": null, "LUISA_TILE_BENCH_DUMP_SOURCE": null }, "artifacts_sha256": { "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx": "9cbdc7873355118a9874c58eec499cdb0692dd9286c14b52afceae620c62ad87", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libSPIRV.16.3.0.dylib": "c0ff450913316a86f6544729b2efc15b23ca1a73994d6b19c20d715ec8646901", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libglslang-default-resource-limits.16.3.0.dylib": "bf2ea542cee8832f0638b0db27ddb23fac6488060a367f2f10c3e98e3fb88999", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libglslang.16.3.0.dylib": "bbdbf823956fb94d1131177bb87c90b82209fdcbaced1466bbf337386048459d", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-ast.dylib": "c90d64cebb541f658c5df764b3e05ec61cffbc884a01662eed43ba22620cfea7", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-backend-metal.so": "e5a069b6b0381058cef33aec8c0ea6788fe5536072f886b36e0253e2258859dd", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-benchmark-metal-timing.dylib": "eb568b9177926113e8af2ea1ec995b9c90c13623fc1e73d483864c0b8217eae0", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-core.dylib": "89ba28beebc6d47fee75f43b1f51e678ba35a0dcf728bb6f4a53ae3791adb87a", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-coro.dylib": "bbfb0d881eb8352ff99b95935619fcd8c36ab9c934c18a541a2118d320496487", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-dsl.dylib": "20ac9be7d1393baea0cb951872134db82057c1a71cd5118b3363a40e8c144ff5", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-osl.dylib": "0239b750e65ce07da91de82175f2577004a3a5cc62746562d73a2cc98cda9929", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-runtime.dylib": "371b29d248be11edba1a5fda9e859b9ac06cc2f87361d74a4bdda375a06cf799", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-tile-bridge-tirx.dylib": "1a06ad07d6d4eeb4d645b6324ea3719b5192940537da786df65860c77754b420", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-tile-bridge-xir.dylib": "77b6343363ba3b2dc242d7c0a22d4b68c277b057a0795f52ba98090200954d95", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-tile.dylib": "3c826ec66f1c3bc303b7a3eaec0641cabc401fc5f9489c99f09201746e57883f", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-validation-layer.so": "d86328e83c026f502b4df69f48d6b3fb756491a4900c244b99900146e812f282", "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-xir.dylib": "9efd84887588f8943c9e0c25cc59eaf45ad4f860f738f9cb61cf817e88a831d9", "/private/tmp/luisa-tvmx-venv/lib/python3.14/site-packages/tvm/lib/libtvm_compiler.dylib": "44a277c13f8400925b6eb7170148b0c0e03ca727a70d29f33713d0bc0c8d5c89", "/private/tmp/luisa-tvmx-venv/lib/python3.14/site-packages/tvm/lib/libtvm_runtime.dylib": "bf5e2c96f4f946e27d590f99d859860d3c6a8a7803b20bbd17cd3534bb759f9d", "/private/tmp/luisa-tvmx-venv/lib/python3.14/site-packages/tvm/lib/libtvm_runtime_metal.dylib": "83dc06e34c4a531f0eb0ab1912579c8ee7c2a05f2736d7cdfd7d7d9ce700ac60", "/private/tmp/luisa-tvmx-venv/lib/python3.14/site-packages/tvm_ffi/lib/libtvm_ffi.dylib": "3dec87d55293c7dea456d66be3d26060b53a769c0518f6c7c18b488dad0fd6cd" }, "capture_sources": true, "metal_device_timing": "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/libluisa-benchmark-metal-timing.dylib", "native_sha256": "9cbdc7873355118a9874c58eec499cdb0692dd9286c14b52afceae620c62ad87", "adjacent_tile_library_sha256": { "libluisa-tile-bridge-tirx.dylib": "1a06ad07d6d4eeb4d645b6324ea3719b5192940537da786df65860c77754b420", "libluisa-tile-bridge-xir.dylib": "77b6343363ba3b2dc242d7c0a22d4b68c277b057a0795f52ba98090200954d95", "libluisa-tile.dylib": "3c826ec66f1c3bc303b7a3eaec0641cabc401fc5f9489c99f09201746e57883f" }, "native_variants": { "reference": { "binary": "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "sha256": "9cbdc7873355118a9874c58eec499cdb0692dd9286c14b52afceae620c62ad87", "adjacent_tile_library_sha256": { "libluisa-tile-bridge-tirx.dylib": "1a06ad07d6d4eeb4d645b6324ea3719b5192940537da786df65860c77754b420", "libluisa-tile-bridge-xir.dylib": "77b6343363ba3b2dc242d7c0a22d4b68c277b057a0795f52ba98090200954d95", "libluisa-tile.dylib": "3c826ec66f1c3bc303b7a3eaec0641cabc401fc5f9489c99f09201746e57883f" } }, "candidate": { "binary": "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "sha256": "9cbdc7873355118a9874c58eec499cdb0692dd9286c14b52afceae620c62ad87", "adjacent_tile_library_sha256": { "libluisa-tile-bridge-tirx.dylib": "1a06ad07d6d4eeb4d645b6324ea3719b5192940537da786df65860c77754b420", "libluisa-tile-bridge-xir.dylib": "77b6343363ba3b2dc242d7c0a22d4b68c277b057a0795f52ba98090200954d95", "libluisa-tile.dylib": "3c826ec66f1c3bc303b7a3eaec0641cabc401fc5f9489c99f09201746e57883f" } } }, "rounds": 4, "samples": 9, "sample_ms": 30, "warmup_ms": 200, "candidate_vector_mode": "reported", "candidate_subgroup_fences": "reported", "candidate_cpu_stack_bytes": null, "shared_cpu_stack_bytes": null, "candidate_cpu_vector_lanes": null, "shared_cpu_vector_lanes": null, "candidate_cpu_input_views": false, "candidate_cpu_model": null, "timing": "synchronized device-resident host wall time including dispatch; no profiler", "source_reports": { "reference": { "path": "scripts/benchmark/tile_torch/results/m1-max-20260905-cooperating-packing/reference/results.json", "sha256": "9db930c33983bd52311fc19417706a59cb4526b1ec4887994489672fbde58cc3" }, "candidate": { "path": "scripts/benchmark/tile_torch/results/m1-max-20260905-cooperating-packing/candidate/results.json", "sha256": "898a666aa7a9b99e42c06ab658958f35e9b5f663277924b880ee2ecd5937c344" } }, "frozen_plans": { "reference": [ { "backend": "metal", "name": "softmax_37x1537", "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_256x3072", "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_768x6144", "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_1024x4096", "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_37x1537", "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_256x3072", "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_768x6144", "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_1024x4096", "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_37x1537", "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_256x3072", "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_768x6144", "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_1024x4096", "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 256, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 1, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" } ], "candidate": [ { "backend": "metal", "name": "softmax_37x1537", "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_256x3072", "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_768x6144", "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "softmax_1024x4096", "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_37x1537", "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_256x3072", "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_768x6144", "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "rmsnorm_1024x4096", "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_37x1537", "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "gemm_block": [ 1, 1537, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_256x3072", "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "gemm_block": [ 1, 3072, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_768x6144", "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "gemm_block": [ 1, 6144, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" }, { "backend": "metal", "name": "layernorm_1024x4096", "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "gemm_block": [ 1, 4096, 1 ], "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "no_vectorize": false, "auto_vectorize": false, "group_threads": 512, "copy_batch": 1, "cpu_stack_bytes": 0, "cpu_vector_lanes": 16, "cpu_input_views": false, "input_views": false, "cpu_model": null, "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "shared_tile_materialization": "preserve", "metal_subgroup_reductions": true, "reduction_programs_per_group": 2, "reduction_unroll": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "element_grid": "auto", "expected_cpu_model": null, "elide_independent_subgroup_barriers": false, "matrix_realization": "simdgroup" } ] }, "artifacts_unchanged": true }, "results": [ { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-bdkk4hy3/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "1ca0ba2a59ee9b200a0f7933e3c823dd873803c26a844f54bebdc043f01bc250", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.454666, "capture_ms": 0.073958, "compile_ms": 55.092542, "allocation_upload_ms": 1.251292, "cold_call_ms": 2.818083, "warmup_ms": 200.451625, "download_ms": 0.502, "repetitions": 4892, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 5.7342260834, 5.79366823385, 5.55178515944, 5.8515944399, 5.58693601799, 5.56814697465, 6.05683605887, 5.59391189697, 5.58516455437 ], "latency_us": [ 256.291, 452.333, 1981.833, 390.667, 307.125, 477.083, 247.916, 213.375, 237.417 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 242833, "compute_span_ns": 242833, "calibration_cpu_ns": 689458, "calibration_gpu_ticks": 689458, "compute_passes": 1, "command_buffer_ns": 242916.692514, "command_buffers": 2 }, { "compute_ns": 241083, "compute_span_ns": 241083, "calibration_cpu_ns": 710333, "calibration_gpu_ticks": 710333, "compute_passes": 1, "command_buffer_ns": 241125.002503, "command_buffers": 2 }, { "compute_ns": 260667, "compute_span_ns": 260667, "calibration_cpu_ns": 729333, "calibration_gpu_ticks": 729333, "compute_passes": 1, "command_buffer_ns": 260750.006419, "command_buffers": 2 }, { "compute_ns": 634958, "compute_span_ns": 634958, "calibration_cpu_ns": 2097458, "calibration_gpu_ticks": 2097458, "compute_passes": 1, "command_buffer_ns": 634999.94576, "command_buffers": 2 }, { "compute_ns": 715625, "compute_span_ns": 715625, "calibration_cpu_ns": 2022334, "calibration_gpu_ticks": 2022334, "compute_passes": 1, "command_buffer_ns": 715666.683391, "command_buffers": 2 }, { "compute_ns": 242709, "compute_span_ns": 242709, "calibration_cpu_ns": 692208, "calibration_gpu_ticks": 692208, "compute_passes": 1, "command_buffer_ns": 242875.015829, "command_buffers": 2 }, { "compute_ns": 282167, "compute_span_ns": 282167, "calibration_cpu_ns": 769750, "calibration_gpu_ticks": 769750, "compute_passes": 1, "command_buffer_ns": 282333.348878, "command_buffers": 2 }, { "compute_ns": 262333, "compute_span_ns": 262333, "calibration_cpu_ns": 829167, "calibration_gpu_ticks": 829167, "compute_passes": 1, "command_buffer_ns": 262374.931481, "command_buffers": 2 }, { "compute_ns": 241666, "compute_span_ns": 241666, "calibration_cpu_ns": 896750, "calibration_gpu_ticks": 896750, "compute_passes": 1, "command_buffer_ns": 241625.006311, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 789250, "calibration_gpu_ticks": 789250, "compute_passes": 1, "command_buffer_ns": 7375.02705306, "command_buffers": 2 }, { "compute_ns": 6083, "compute_span_ns": 6083, "calibration_cpu_ns": 426083, "calibration_gpu_ticks": 426083, "compute_passes": 1, "command_buffer_ns": 6166.69422016, "command_buffers": 2 }, { "compute_ns": 9125, "compute_span_ns": 9125, "calibration_cpu_ns": 1193500, "calibration_gpu_ticks": 1193500, "compute_passes": 1, "command_buffer_ns": 9166.65885597, "command_buffers": 2 }, { "compute_ns": 7375, "compute_span_ns": 7375, "calibration_cpu_ns": 773666, "calibration_gpu_ticks": 773666, "compute_passes": 1, "command_buffer_ns": 7500.0571087, "command_buffers": 2 }, { "compute_ns": 30167, "compute_span_ns": 30167, "calibration_cpu_ns": 1327791, "calibration_gpu_ticks": 1327791, "compute_passes": 1, "command_buffer_ns": 30249.9393001, "command_buffers": 2 }, { "compute_ns": 7375, "compute_span_ns": 7375, "calibration_cpu_ns": 941125, "calibration_gpu_ticks": 941125, "compute_passes": 1, "command_buffer_ns": 7416.70373827, "command_buffers": 2 }, { "compute_ns": 61750, "compute_span_ns": 61750, "calibration_cpu_ns": 1241583, "calibration_gpu_ticks": 1241583, "compute_passes": 1, "command_buffer_ns": 61833.3579041, "command_buffers": 2 }, { "compute_ns": 7333, "compute_span_ns": 7333, "calibration_cpu_ns": 773334, "calibration_gpu_ticks": 773334, "compute_passes": 1, "command_buffer_ns": 7374.9688454, "command_buffers": 2 }, { "compute_ns": 26416, "compute_span_ns": 26416, "calibration_cpu_ns": 1375416, "calibration_gpu_ticks": 1375416, "compute_passes": 1, "command_buffer_ns": 26458.3504759, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 508375.000209, "command_buffers": 2 }, { "command_buffer_ns": 646583.328489, "command_buffers": 2 }, { "command_buffer_ns": 262124.987785, "command_buffers": 2 }, { "command_buffer_ns": 242708.367296, "command_buffers": 2 }, { "command_buffer_ns": 240624.998696, "command_buffers": 2 }, { "command_buffer_ns": 240791.705437, "command_buffers": 2 }, { "command_buffer_ns": 679749.937262, "command_buffers": 2 }, { "command_buffer_ns": 621958.286501, "command_buffers": 2 }, { "command_buffer_ns": 241583.271418, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 9749.95782599, "command_buffers": 2 }, { "command_buffer_ns": 13958.2552947, "command_buffers": 2 }, { "command_buffer_ns": 6708.37471262, "command_buffers": 2 }, { "command_buffer_ns": 6083.34084973, "command_buffers": 2 }, { "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "command_buffer_ns": 6250.04759058, "command_buffers": 2 }, { "command_buffer_ns": 6416.69612378, "command_buffers": 2 }, { "command_buffer_ns": 6208.31269771, "command_buffers": 2 }, { "command_buffer_ns": 6666.63981974, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 7.943359378265625, 10.102864507640625, 4.0957029341406255, 3.792318239, 3.759765604625, 3.762370397453125, 10.62109276971875, 9.718098226578126, 3.7747386159062497 ], "command_buffer_throughput_us_p50": 4.0957029341406255, "command_buffer_throughput_us_p90": 10.20651016005625, "command_buffer_latency_us": [ 9.74995782599, 13.9582552947, 6.7083747126199995, 6.08334084973, 6.49999128655, 6.25004759058, 6.4166961237799995, 6.20831269771, 6.66663981974 ], "command_buffer_latency_us_p50": 6.49999128655, "command_buffer_latency_us_p90": 10.591617319732 }, "compute_throughput_us": [ 3.794265625, 3.766921875, 4.072921875, 9.92121875, 11.181640625, 3.792328125, 4.408859375, 4.098953125, 3.77603125 ], "compute_throughput_us_p50": 4.072921875, "compute_throughput_us_p90": 10.173303125, "command_buffer_throughput_us": [ 3.79557332053125, 3.767578164109375, 4.074218850296875, 9.921874152500001, 11.182291927984375, 3.794922122328125, 4.41145857621875, 4.099608304390625, 3.775390723609375 ], "command_buffer_throughput_us_p50": 4.074218850296875, "command_buffer_throughput_us_p90": 10.173957707596877, "compute_latency_us": [ 7.25, 6.083, 9.125, 7.375, 30.167, 7.375, 61.75, 7.333, 26.416 ], "compute_latency_us_p50": 7.375, "compute_latency_us_p90": 36.48360000000001, "command_buffer_latency_us": [ 7.37502705306, 6.16669422016, 9.166658855969999, 7.5000571087, 30.2499393001, 7.41670373827, 61.8333579041, 7.3749688454, 26.458350475899998 ], "command_buffer_latency_us_p50": 7.5000571087, "command_buffer_latency_us_p90": 36.5666230209, "counter_control_throughput_ratio": 0.9947544819072046, "counter_control_latency_ratio": 1.2328845767425443 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 5.59391189697, "throughput_us_p90": 5.892642763694, "latency_us_p50": 307.125, "latency_us_p90": 778.0330000000004 }, "torch": { "cold_call_ms": 52.690458, "warmup_ms": 201.158834, "repetitions": 884, "throughput_us": [ 32.890789592760186, 31.201923076923077, 30.148472850678733, 33.456259049773756, 30.68627488687783, 29.66963574660634, 28.952064479638008, 32.25358257918552, 30.46431900452489 ], "latency_us": [ 344.708, 402.167, 340.667, 311.0, 321.0, 345.04200000000003, 330.667, 2338.834, 581.0419999999999 ], "throughput_us_p50": 30.68627488687783, "throughput_us_p90": 33.0038834841629, "latency_us_p50": 344.708, "latency_us_p90": 932.6004000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1053083.3969824016, "command_buffers": 7 }, { "command_buffer_ns": 1075541.7170003057, "command_buffers": 7 }, { "command_buffer_ns": 1039749.8263046145, "command_buffers": 7 }, { "command_buffer_ns": 1139624.9756217003, "command_buffers": 7 }, { "command_buffer_ns": 1040291.681420058, "command_buffers": 7 }, { "command_buffer_ns": 1071541.6283346713, "command_buffers": 7 }, { "command_buffer_ns": 1040374.9765828252, "command_buffers": 7 }, { "command_buffer_ns": 1099000.044632703, "command_buffers": 7 }, { "command_buffer_ns": 1038958.3185315132, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 85000.0069476664, "command_buffers": 1 }, { "command_buffer_ns": 53833.29698815942, "command_buffers": 1 }, { "command_buffer_ns": 71458.3438821137, "command_buffers": 1 }, { "command_buffer_ns": 54000.00372901559, "command_buffers": 1 }, { "command_buffer_ns": 131916.66221246123, "command_buffers": 1 }, { "command_buffer_ns": 57625.0022277236, "command_buffers": 1 }, { "command_buffer_ns": 57958.357501775026, "command_buffers": 1 }, { "command_buffer_ns": 54124.9755769968, "command_buffers": 1 }, { "command_buffer_ns": 100166.7114906013, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.454428077850025, 16.805339328129776, 16.246091036009602, 17.806640244089067, 16.254557522188406, 16.74283794272924, 16.255859009106643, 17.171875697385985, 16.233723727054894 ], "command_buffer_throughput_us_p50": 16.454428077850025, "command_buffer_throughput_us_p90": 17.2988286067266, "command_buffer_latency_us": [ 85.0000069476664, 53.83329698815942, 71.4583438821137, 54.00000372901559, 131.91666221246123, 57.6250022277236, 57.958357501775026, 54.1249755769968, 100.1667114906013 ], "command_buffer_latency_us_p50": 57.958357501775026, "command_buffer_latency_us_p90": 106.51670163497329 }, "throughput": [ { "compute_ns": 3486751.0, "compute_span_ns": 4592417.0, "command_buffer_ns": 4423375.008627772, "calibration_cpu_ns": 6099209.0, "calibration_gpu_ticks": 6099209.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3495542.0, "compute_span_ns": 4603667.0, "command_buffer_ns": 4447125.014849007, "calibration_cpu_ns": 6084458.0, "calibration_gpu_ticks": 6084458.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3513044.0, "compute_span_ns": 4719458.0, "command_buffer_ns": 4604750.080034137, "calibration_cpu_ns": 6055792.0, "calibration_gpu_ticks": 6055792.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3501080.0, "compute_span_ns": 4596375.0, "command_buffer_ns": 4448875.0281743705, "calibration_cpu_ns": 6220084.0, "calibration_gpu_ticks": 6220084.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3483668.0, "compute_span_ns": 4593541.0, "command_buffer_ns": 4490458.290092647, "calibration_cpu_ns": 6128250.0, "calibration_gpu_ticks": 6128250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3523911.0, "compute_span_ns": 4511791.0, "command_buffer_ns": 4417999.996803701, "calibration_cpu_ns": 6022583.0, "calibration_gpu_ticks": 6022583.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3496584.0, "compute_span_ns": 4649875.0, "command_buffer_ns": 4550625.046249479, "calibration_cpu_ns": 6209750.0, "calibration_gpu_ticks": 6209750.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3500957.0, "compute_span_ns": 4580042.0, "command_buffer_ns": 4474791.639950126, "calibration_cpu_ns": 6097667.0, "calibration_gpu_ticks": 6097667.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3498997.0, "compute_span_ns": 4568042.0, "command_buffer_ns": 4436916.555278003, "calibration_cpu_ns": 6207250.0, "calibration_gpu_ticks": 6207250.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 59083.0, "compute_span_ns": 59083.0, "command_buffer_ns": 59083.33696424961, "calibration_cpu_ns": 962958.0, "calibration_gpu_ticks": 962958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56833.0, "compute_span_ns": 56833.0, "command_buffer_ns": 56833.31983163953, "calibration_cpu_ns": 564917.0, "calibration_gpu_ticks": 564917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 54750.0, "compute_span_ns": 54750.0, "command_buffer_ns": 54750.009439885616, "calibration_cpu_ns": 957291.0, "calibration_gpu_ticks": 957291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 54709.0, "compute_span_ns": 54709.0, "command_buffer_ns": 54750.009439885616, "calibration_cpu_ns": 531375.0, "calibration_gpu_ticks": 531375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 53917.0, "compute_span_ns": 53917.0, "command_buffer_ns": 53958.327043801546, "calibration_cpu_ns": 964875.0, "calibration_gpu_ticks": 964875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 57000.0, "compute_span_ns": 57000.0, "command_buffer_ns": 57000.0265724957, "calibration_cpu_ns": 504083.0, "calibration_gpu_ticks": 504083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 58125.0, "compute_span_ns": 58125.0, "command_buffer_ns": 58125.00603497028, "calibration_cpu_ns": 558792.0, "calibration_gpu_ticks": 558792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 55375.0, "compute_span_ns": 55375.0, "command_buffer_ns": 55374.985095113516, "calibration_cpu_ns": 603667.0, "calibration_gpu_ticks": 603667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56916.0, "compute_span_ns": 56916.0, "command_buffer_ns": 56874.99651685357, "calibration_cpu_ns": 925125.0, "calibration_gpu_ticks": 925125.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 54.480484375, 54.61784375, 54.8913125, 54.704375, 54.4323125, 55.061109375, 54.634125, 54.702453125, 54.671828125 ], "compute_throughput_us_p50": 54.671828125, "compute_throughput_us_p90": 54.925271875, "command_buffer_throughput_us": [ 69.11523450980894, 69.48632835701574, 71.94922000053339, 69.51367231522454, 70.16341078269761, 69.03124995005783, 71.10351634764811, 69.91861937422073, 69.3268211762188 ], "command_buffer_throughput_us_p50": 69.51367231522454, "command_buffer_throughput_us_p90": 71.27265707822517, "compute_latency_us": [ 59.083, 56.833, 54.75, 54.709, 53.917, 57.0, 58.125, 55.375, 56.916 ], "compute_latency_us_p50": 56.833, "compute_latency_us_p90": 58.3166, "command_buffer_latency_us": [ 59.08333696424961, 56.83331983163953, 54.750009439885616, 54.750009439885616, 53.958327043801546, 57.0000265724957, 58.12500603497028, 55.374985095113516, 56.87499651685357 ], "command_buffer_latency_us_p50": 56.83331983163953, "command_buffer_latency_us_p90": 58.31667222082615, "counter_control_throughput_ratio": 4.200403331115942, "counter_control_latency_ratio": 0.9891544359033929 }, "download_ms": 0.315709, "allocation_upload_ms": 4.196667, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.182293612293817, "device_compute_slowdown": 0.07449763460786048, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 1.96275, "warmup_ms": 200.669417, "repetitions": 913, "throughput_us": [ 33.62267250821468, 31.059191675794086, 33.51620153340635, 30.31147316538883, 32.62732749178532, 30.261318729463305, 29.285779846659366, 31.65197152245345, 37.89882256297919 ], "latency_us": [ 350.95799999999997, 360.625, 356.45799999999997, 327.458, 337.75, 313.625, 418.833, 292.66700000000003, 300.875 ], "throughput_us_p50": 31.65197152245345, "throughput_us_p90": 34.47790251916758, "latency_us_p50": 337.75, "latency_us_p90": 372.26660000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1080750.022083521, "command_buffers": 7 }, { "command_buffer_ns": 1023291.7265966535, "command_buffers": 7 }, { "command_buffer_ns": 1029583.4508724511, "command_buffers": 7 }, { "command_buffer_ns": 1025750.0107400119, "command_buffers": 7 }, { "command_buffer_ns": 1041458.3957754076, "command_buffers": 7 }, { "command_buffer_ns": 1049416.6053831577, "command_buffers": 7 }, { "command_buffer_ns": 1020708.295982331, "command_buffers": 7 }, { "command_buffer_ns": 1034500.1355744898, "command_buffers": 7 }, { "command_buffer_ns": 1021500.036586076, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 60458.31833034754, "command_buffers": 1 }, { "command_buffer_ns": 54583.302699029446, "command_buffers": 1 }, { "command_buffer_ns": 54000.00372901559, "command_buffers": 1 }, { "command_buffer_ns": 58749.98169019818, "command_buffers": 1 }, { "command_buffer_ns": 59375.015553086996, "command_buffers": 1 }, { "command_buffer_ns": 54500.007536262274, "command_buffers": 1 }, { "command_buffer_ns": 57708.355598151684, "command_buffers": 1 }, { "command_buffer_ns": 55374.985095113516, "command_buffers": 1 }, { "command_buffer_ns": 54583.302699029446, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.886719095055014, 15.98893322807271, 16.087241419882048, 16.027343917812686, 16.272787433990743, 16.39713445911184, 15.948567124723922, 16.164064618351404, 15.960938071657438 ], "command_buffer_throughput_us_p50": 16.087241419882048, "command_buffer_throughput_us_p90": 16.495051386300474, "command_buffer_latency_us": [ 60.45831833034754, 54.583302699029446, 54.00000372901559, 58.74998169019818, 59.375015553086996, 54.500007536262274, 57.708355598151684, 55.374985095113516, 54.583302699029446 ], "command_buffer_latency_us_p50": 55.374985095113516, "command_buffer_latency_us_p90": 59.591676108539104 }, "throughput": [ { "compute_ns": 3527793.0, "compute_span_ns": 4484125.0, "command_buffer_ns": 4385791.718959808, "calibration_cpu_ns": 6479250.0, "calibration_gpu_ticks": 6479250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3524499.0, "compute_span_ns": 4857291.0, "command_buffer_ns": 4773875.058162957, "calibration_cpu_ns": 6858083.0, "calibration_gpu_ticks": 6858083.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3667836.0, "compute_span_ns": 4680833.0, "command_buffer_ns": 4594374.971929938, "calibration_cpu_ns": 6590083.0, "calibration_gpu_ticks": 6590083.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3494247.0, "compute_span_ns": 4543833.0, "command_buffer_ns": 4436124.989297241, "calibration_cpu_ns": 6599250.0, "calibration_gpu_ticks": 6599250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3475039.0, "compute_span_ns": 4288375.0, "command_buffer_ns": 4195624.904241413, "calibration_cpu_ns": 6120458.0, "calibration_gpu_ticks": 6120458.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 5136918.0, "compute_span_ns": 6165917.0, "command_buffer_ns": 5839749.99608472, "calibration_cpu_ns": 9992541.0, "calibration_gpu_ticks": 9992541.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3513667.0, "compute_span_ns": 4461083.0, "command_buffer_ns": 4378916.695713997, "calibration_cpu_ns": 6887083.0, "calibration_gpu_ticks": 6887083.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3637044.0, "compute_span_ns": 4747084.0, "command_buffer_ns": 4659333.382733166, "calibration_cpu_ns": 7065375.0, "calibration_gpu_ticks": 7065375.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3727796.0, "compute_span_ns": 4743833.0, "command_buffer_ns": 4642250.016331673, "calibration_cpu_ns": 7429291.0, "calibration_gpu_ticks": 7429291.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 53417.0, "compute_span_ns": 53417.0, "command_buffer_ns": 53458.32323655486, "calibration_cpu_ns": 843375.0, "calibration_gpu_ticks": 843375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 54042.0, "compute_span_ns": 54042.0, "command_buffer_ns": 54083.29889178276, "calibration_cpu_ns": 620209.0, "calibration_gpu_ticks": 620209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56333.0, "compute_span_ns": 56333.0, "command_buffer_ns": 56333.37423205376, "calibration_cpu_ns": 3197417.0, "calibration_gpu_ticks": 3197417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 61167.0, "compute_span_ns": 61167.0, "command_buffer_ns": 61166.705563664436, "calibration_cpu_ns": 1862166.0, "calibration_gpu_ticks": 1862166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 53625.0, "compute_span_ns": 53625.0, "command_buffer_ns": 53624.97176975012, "calibration_cpu_ns": 871958.0, "calibration_gpu_ticks": 871958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56167.0, "compute_span_ns": 56167.0, "command_buffer_ns": 56166.667491197586, "calibration_cpu_ns": 1986584.0, "calibration_gpu_ticks": 1986584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 60584.0, "compute_span_ns": 60584.0, "command_buffer_ns": 60625.02507120371, "calibration_cpu_ns": 1863750.0, "calibration_gpu_ticks": 1863750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 53625.0, "compute_span_ns": 53625.0, "command_buffer_ns": 53625.02997741103, "calibration_cpu_ns": 847333.0, "calibration_gpu_ticks": 847333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 61666.0, "compute_span_ns": 61666.0, "command_buffer_ns": 61624.974478036165, "calibration_cpu_ns": 1969083.0, "calibration_gpu_ticks": 1969083.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 55.121765625, 55.070296875, 57.3099375, 54.597609375, 54.297484375, 80.26434375, 54.901046875, 56.8288125, 58.2468125 ], "compute_throughput_us_p50": 55.121765625, "compute_throughput_us_p90": 62.65031875, "command_buffer_throughput_us": [ 68.527995608747, 74.59179778379621, 71.78710893640528, 69.31445295776939, 65.55663912877208, 91.24609368882375, 68.4205733705312, 72.80208410520572, 72.53515650518239 ], "command_buffer_throughput_us_p50": 71.78710893640528, "command_buffer_throughput_us_p90": 77.92265696480172, "compute_latency_us": [ 53.417, 54.042, 56.333, 61.167, 53.625, 56.167, 60.584, 53.625, 61.666 ], "compute_latency_us_p50": 56.167, "compute_latency_us_p90": 61.2668, "command_buffer_latency_us": [ 53.45832323655486, 54.08329889178276, 56.33337423205376, 61.166705563664436, 53.62497176975012, 56.166667491197586, 60.62502507120371, 53.62502997741103, 61.624974478036165 ], "command_buffer_latency_us_p50": 56.166667491197586, "command_buffer_latency_us_p90": 61.25835934653878, "counter_control_throughput_ratio": 4.462362878926176, "counter_control_latency_ratio": 1.0305809123755878 }, "download_ms": 0.630542, "allocation_upload_ms": 0.920625, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-a5we3h6i/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "d45e9e72129e03853b1c708b1bdcbc59be9547b0d0e02ca6bdc2eb0b41c08926", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.545125, "capture_ms": 0.079458, "compile_ms": 60.768416, "allocation_upload_ms": 1.039333, "cold_call_ms": 62.879958, "warmup_ms": 200.001917, "download_ms": 0.416375, "repetitions": 4941, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.49815320785, 6.48848067193, 5.96120056669, 6.32830722526, 6.48353066181, 6.47326796195, 5.95102206031, 6.5340856102, 6.42978830196 ], "latency_us": [ 279.625, 229.375, 216.708, 190.084, 208.125, 208.084, 211, 201.667, 191.167 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 263666, "compute_span_ns": 263666, "calibration_cpu_ns": 744833, "calibration_gpu_ticks": 744833, "compute_passes": 1, "command_buffer_ns": 263624.999207, "command_buffers": 2 }, { "compute_ns": 641917, "compute_span_ns": 641917, "calibration_cpu_ns": 2036208, "calibration_gpu_ticks": 2036208, "compute_passes": 1, "command_buffer_ns": 641958.264168, "command_buffers": 2 }, { "compute_ns": 287709, "compute_span_ns": 287709, "calibration_cpu_ns": 857458, "calibration_gpu_ticks": 857458, "compute_passes": 1, "command_buffer_ns": 287791.597657, "command_buffers": 2 }, { "compute_ns": 264875, "compute_span_ns": 264875, "calibration_cpu_ns": 693500, "calibration_gpu_ticks": 693500, "compute_passes": 1, "command_buffer_ns": 264875.008725, "command_buffers": 2 }, { "compute_ns": 792250, "compute_span_ns": 792250, "calibration_cpu_ns": 2452916, "calibration_gpu_ticks": 2452916, "compute_passes": 1, "command_buffer_ns": 792333.332356, "command_buffers": 2 }, { "compute_ns": 285208, "compute_span_ns": 285208, "calibration_cpu_ns": 861000, "calibration_gpu_ticks": 861000, "compute_passes": 1, "command_buffer_ns": 285208.341666, "command_buffers": 2 }, { "compute_ns": 265750, "compute_span_ns": 265750, "calibration_cpu_ns": 716500, "calibration_gpu_ticks": 716500, "compute_passes": 1, "command_buffer_ns": 265750.044491, "command_buffers": 2 }, { "compute_ns": 658750, "compute_span_ns": 658750, "calibration_cpu_ns": 1866333, "calibration_gpu_ticks": 1866333, "compute_passes": 1, "command_buffer_ns": 658875.040244, "command_buffers": 2 }, { "compute_ns": 627250, "compute_span_ns": 627250, "calibration_cpu_ns": 1826875, "calibration_gpu_ticks": 1826875, "compute_passes": 1, "command_buffer_ns": 627291.62164, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 751333, "calibration_gpu_ticks": 751333, "compute_passes": 1, "command_buffer_ns": 7041.72998667, "command_buffers": 2 }, { "compute_ns": 48292, "compute_span_ns": 48292, "calibration_cpu_ns": 1004333, "calibration_gpu_ticks": 1004333, "compute_passes": 1, "command_buffer_ns": 48458.3433717, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 448459, "calibration_gpu_ticks": 448459, "compute_passes": 1, "command_buffer_ns": 7291.67368263, "command_buffers": 2 }, { "compute_ns": 88083, "compute_span_ns": 88083, "calibration_cpu_ns": 1101500, "calibration_gpu_ticks": 1101500, "compute_passes": 1, "command_buffer_ns": 88208.2968019, "command_buffers": 2 }, { "compute_ns": 7208, "compute_span_ns": 7208, "calibration_cpu_ns": 461583, "calibration_gpu_ticks": 461583, "compute_passes": 1, "command_buffer_ns": 7291.61547497, "command_buffers": 2 }, { "compute_ns": 22666, "compute_span_ns": 22666, "calibration_cpu_ns": 1400875, "calibration_gpu_ticks": 1400875, "compute_passes": 1, "command_buffer_ns": 22749.9986067, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 410042, "calibration_gpu_ticks": 410042, "compute_passes": 1, "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "compute_ns": 66625, "compute_span_ns": 66625, "calibration_cpu_ns": 1373541, "calibration_gpu_ticks": 1373541, "compute_passes": 1, "command_buffer_ns": 66708.3077133, "command_buffers": 2 }, { "compute_ns": 165208, "compute_span_ns": 165208, "calibration_cpu_ns": 1412000, "calibration_gpu_ticks": 1412000, "compute_passes": 1, "command_buffer_ns": 165291.596204, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 492250.022944, "command_buffers": 2 }, { "command_buffer_ns": 276041.624602, "command_buffers": 2 }, { "command_buffer_ns": 833833.299112, "command_buffers": 2 }, { "command_buffer_ns": 746708.305087, "command_buffers": 2 }, { "command_buffer_ns": 291958.334856, "command_buffers": 2 }, { "command_buffer_ns": 771125.021856, "command_buffers": 2 }, { "command_buffer_ns": 295166.682918, "command_buffers": 2 }, { "command_buffer_ns": 299291.685224, "command_buffers": 2 }, { "command_buffer_ns": 265791.662969, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 11291.7041406, "command_buffers": 2 }, { "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "command_buffer_ns": 6583.34465697, "command_buffers": 2 }, { "command_buffer_ns": 7583.35227147, "command_buffers": 2 }, { "command_buffer_ns": 6750.05139783, "command_buffers": 2 }, { "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "command_buffer_ns": 7625.02895668, "command_buffers": 2 }, { "command_buffer_ns": 6625.02134219, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 7.6914066085, 4.31315038440625, 13.028645298625001, 11.667317266984375, 4.5618489821249995, 12.0488284665, 4.61197942059375, 4.676432581625, 4.152994733890625 ], "command_buffer_throughput_us_p50": 4.676432581625, "command_buffer_throughput_us_p90": 12.244791832925, "command_buffer_latency_us": [ 11.2917041406, 7.66664743423, 6.58334465697, 7.58335227147, 6.750051397829999, 7.54167558625, 6.54166797176, 7.6250289566800005, 6.62502134219 ], "command_buffer_latency_us_p50": 7.54167558625, "command_buffer_latency_us_p90": 8.391658775504 }, "compute_throughput_us": [ 4.11978125, 10.029953125, 4.495453125, 4.138671875, 12.37890625, 4.456375, 4.15234375, 10.29296875, 9.80078125 ], "compute_throughput_us_p50": 4.495453125, "compute_throughput_us_p90": 10.71015625, "command_buffer_throughput_us": [ 4.1191406126093755, 10.030597877624999, 4.496743713390625, 4.138672011328126, 12.380208318062499, 4.45638033853125, 4.152344445171875, 10.2949225038125, 9.801431588125 ], "command_buffer_throughput_us_p50": 4.496743713390625, "command_buffer_throughput_us_p90": 10.7119796666625, "compute_latency_us": [ 7.0, 48.292, 7.25, 88.083, 7.208, 22.666, 7.125, 66.625, 165.208 ], "compute_latency_us_p50": 22.666, "compute_latency_us_p90": 103.50800000000001, "command_buffer_latency_us": [ 7.041729986669999, 48.4583433717, 7.29167368263, 88.20829680189999, 7.2916154749699995, 22.7499986067, 7.12502514943, 66.7083077133, 165.291596204 ], "command_buffer_latency_us_p50": 22.7499986067, "command_buffer_latency_us_p90": 103.62495668232, "counter_control_throughput_ratio": 0.9003388928039274, "counter_control_latency_ratio": 3.0165708331684074 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 6.47326796195, "throughput_us_p90": 6.505339688319999, "latency_us_p50": 208.125, "latency_us_p90": 239.425 }, "slowdown": 0.20451389441437978, "device_compute_slowdown": 0.0815549551801934, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-i85kc8hw/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7bac43b2781c01105a8e678fa6161555c0e89cbd2b0a98679fdc42bb1bd74b45", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 34.507542, "capture_ms": 0.075417, "compile_ms": 100.677542, "allocation_upload_ms": 1.788459, "cold_call_ms": 6.088709, "warmup_ms": 201.514125, "download_ms": 0.642209, "repetitions": 1370, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 20.0888080292, 22.1710766423, 20.191910219, 21.0010335766, 20.8618613139, 20.3702554745, 21.7501824818, 20.0105226277, 22.0835459854 ], "latency_us": [ 275.875, 234.666, 258.167, 240.958, 224.5, 213.416, 240.917, 234.417, 221.583 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1378708, "compute_span_ns": 1378708, "calibration_cpu_ns": 2433375, "calibration_gpu_ticks": 2433375, "compute_passes": 1, "command_buffer_ns": 1378791.63958, "command_buffers": 2 }, { "compute_ns": 919834, "compute_span_ns": 919834, "calibration_cpu_ns": 1414167, "calibration_gpu_ticks": 1414167, "compute_passes": 1, "command_buffer_ns": 919999.962207, "command_buffers": 2 }, { "compute_ns": 1468125, "compute_span_ns": 1468125, "calibration_cpu_ns": 2774000, "calibration_gpu_ticks": 2774000, "compute_passes": 1, "command_buffer_ns": 1468166.65074, "command_buffers": 2 }, { "compute_ns": 941625, "compute_span_ns": 941625, "calibration_cpu_ns": 1470417, "calibration_gpu_ticks": 1470417, "compute_passes": 1, "command_buffer_ns": 941708.392929, "command_buffers": 2 }, { "compute_ns": 1488750, "compute_span_ns": 1488750, "calibration_cpu_ns": 2809875, "calibration_gpu_ticks": 2809875, "compute_passes": 1, "command_buffer_ns": 1488833.28075, "command_buffers": 2 }, { "compute_ns": 901041, "compute_span_ns": 901041, "calibration_cpu_ns": 1410833, "calibration_gpu_ticks": 1410833, "compute_passes": 1, "command_buffer_ns": 901125.02221, "command_buffers": 2 }, { "compute_ns": 1457166, "compute_span_ns": 1457166, "calibration_cpu_ns": 2809417, "calibration_gpu_ticks": 2809417, "compute_passes": 1, "command_buffer_ns": 1457166.68339, "command_buffers": 2 }, { "compute_ns": 904917, "compute_span_ns": 904917, "calibration_cpu_ns": 1459583, "calibration_gpu_ticks": 1459583, "compute_passes": 1, "command_buffer_ns": 905041.64109, "command_buffers": 2 }, { "compute_ns": 1301500, "compute_span_ns": 1301500, "calibration_cpu_ns": 2362959, "calibration_gpu_ticks": 2362959, "compute_passes": 1, "command_buffer_ns": 1301541.63344, "command_buffers": 2 } ], "latency": [ { "compute_ns": 91500, "compute_span_ns": 91500, "calibration_cpu_ns": 1086000, "calibration_gpu_ticks": 1086000, "compute_passes": 1, "command_buffer_ns": 91541.6749194, "command_buffers": 2 }, { "compute_ns": 18083, "compute_span_ns": 18083, "calibration_cpu_ns": 985917, "calibration_gpu_ticks": 985917, "compute_passes": 1, "command_buffer_ns": 18083.3158083, "command_buffers": 2 }, { "compute_ns": 51042, "compute_span_ns": 51042, "calibration_cpu_ns": 1351041, "calibration_gpu_ticks": 1351041, "compute_passes": 1, "command_buffer_ns": 51208.3643116, "command_buffers": 2 }, { "compute_ns": 20292, "compute_span_ns": 20292, "calibration_cpu_ns": 961791, "calibration_gpu_ticks": 961791, "compute_passes": 1, "command_buffer_ns": 20374.9514185, "command_buffers": 2 }, { "compute_ns": 69583, "compute_span_ns": 69583, "calibration_cpu_ns": 1524916, "calibration_gpu_ticks": 1524916, "compute_passes": 1, "command_buffer_ns": 69583.3587088, "command_buffers": 2 }, { "compute_ns": 20625, "compute_span_ns": 20625, "calibration_cpu_ns": 763792, "calibration_gpu_ticks": 763792, "compute_passes": 1, "command_buffer_ns": 20624.9533221, "command_buffers": 2 }, { "compute_ns": 20917, "compute_span_ns": 20917, "calibration_cpu_ns": 1042583, "calibration_gpu_ticks": 1042583, "compute_passes": 1, "command_buffer_ns": 20958.3668038, "command_buffers": 2 }, { "compute_ns": 16167, "compute_span_ns": 16167, "calibration_cpu_ns": 623917, "calibration_gpu_ticks": 623917, "compute_passes": 1, "command_buffer_ns": 16249.9491125, "command_buffers": 2 }, { "compute_ns": 18958, "compute_span_ns": 18958, "calibration_cpu_ns": 546583, "calibration_gpu_ticks": 546583, "compute_passes": 1, "command_buffer_ns": 19083.3816305, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1224624.98257, "command_buffers": 2 }, { "command_buffer_ns": 1154166.6463, "command_buffers": 2 }, { "command_buffer_ns": 897791.7023, "command_buffers": 2 }, { "command_buffer_ns": 1441625.0051, "command_buffers": 2 }, { "command_buffer_ns": 926666.602027, "command_buffers": 2 }, { "command_buffer_ns": 1287958.3519, "command_buffers": 2 }, { "command_buffer_ns": 922083.389014, "command_buffers": 2 }, { "command_buffer_ns": 1476916.65916, "command_buffers": 2 }, { "command_buffer_ns": 922708.364669, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 16083.358787, "command_buffers": 2 }, { "command_buffer_ns": 16166.6539498, "command_buffers": 2 }, { "command_buffer_ns": 29416.6966341, "command_buffers": 2 }, { "command_buffer_ns": 16250.0073202, "command_buffers": 2 }, { "command_buffer_ns": 18666.672986, "command_buffers": 2 }, { "command_buffer_ns": 16166.7121574, "command_buffers": 2 }, { "command_buffer_ns": 69000.0015311, "command_buffers": 2 }, { "command_buffer_ns": 109958.346002, "command_buffers": 2 }, { "command_buffer_ns": 20833.3367482, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 19.13476535265625, 18.0338538484375, 14.0279953484375, 22.5253907046875, 14.479165656671876, 20.124349248437497, 14.40755295334375, 23.076822799375, 14.417318197953126 ], "command_buffer_throughput_us_p50": 18.0338538484375, "command_buffer_throughput_us_p90": 22.635677123625, "command_buffer_latency_us": [ 16.083358786999998, 16.1666539498, 29.4166966341, 16.2500073202, 18.666672986000002, 16.1667121574, 69.0000015311, 109.958346002, 20.8333367482 ], "command_buffer_latency_us_p50": 18.666672986000002, "command_buffer_latency_us_p90": 77.19167042528001 }, "compute_throughput_us": [ 21.5423125, 14.37240625, 22.939453125, 14.712890625, 23.26171875, 14.078765625, 22.76821875, 14.139328125, 20.3359375 ], "compute_throughput_us_p50": 20.3359375, "compute_throughput_us_p90": 23.00390625, "command_buffer_throughput_us": [ 21.5436193684375, 14.374999409484374, 22.9401039178125, 14.714193639515624, 23.263020011718748, 14.08007847203125, 22.76822942796875, 14.141275642031252, 20.3365880225 ], "command_buffer_throughput_us_p50": 20.3365880225, "command_buffer_throughput_us_p90": 23.00468713659375, "compute_latency_us": [ 91.5, 18.083, 51.042, 20.292, 69.583, 20.625, 20.917, 16.167, 18.958 ], "compute_latency_us_p50": 20.625, "compute_latency_us_p90": 73.96640000000001, "command_buffer_latency_us": [ 91.5416749194, 18.0833158083, 51.2083643116, 20.3749514185, 69.5833587088, 20.624953322099998, 20.9583668038, 16.2499491125, 19.0833816305 ], "command_buffer_latency_us_p50": 20.624953322099998, "command_buffer_latency_us_p90": 73.97502195092001, "counter_control_throughput_ratio": 1.1258888714539088, "counter_control_latency_ratio": 1.2538425993920863 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 20.8618613139, "throughput_us_p90": 22.10105211678, "latency_us_p50": 234.666, "latency_us_p90": 261.7086 }, "torch": { "cold_call_ms": 6.969041, "warmup_ms": 200.304959, "repetitions": 542, "throughput_us": [ 44.799276752767526, 45.68373431734318, 46.19072878228782, 46.43888376383764, 45.9330405904059, 46.379612546125465, 45.52775092250923, 45.66013099630997, 45.70940959409594 ], "latency_us": [ 517.4159999999999, 327.41700000000003, 2321.125, 671.959, 366.125, 570.125, 487.79200000000003, 378.33299999999997, 373.042 ], "throughput_us_p50": 45.70940959409594, "throughput_us_p90": 46.3914667896679, "latency_us_p50": 487.79200000000003, "latency_us_p90": 1001.7922000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2041250.0016391277, "command_buffers": 11 }, { "command_buffer_ns": 1931624.8944960535, "command_buffers": 11 }, { "command_buffer_ns": 2017625.025473535, "command_buffers": 11 }, { "command_buffer_ns": 1955625.0772438943, "command_buffers": 11 }, { "command_buffer_ns": 1872125.1981332898, "command_buffers": 11 }, { "command_buffer_ns": 1920458.220411092, "command_buffers": 11 }, { "command_buffer_ns": 1885291.6546165943, "command_buffers": 11 }, { "command_buffer_ns": 1895041.6706502438, "command_buffers": 11 }, { "command_buffer_ns": 1888708.269689232, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 71041.6934452951, "command_buffers": 1 }, { "command_buffer_ns": 63749.96155500412, "command_buffers": 1 }, { "command_buffer_ns": 49875.001423060894, "command_buffers": 1 }, { "command_buffer_ns": 65250.03118440509, "command_buffers": 1 }, { "command_buffer_ns": 52916.70095175505, "command_buffers": 1 }, { "command_buffer_ns": 49041.700549423695, "command_buffers": 1 }, { "command_buffer_ns": 49541.646149009466, "command_buffers": 1 }, { "command_buffer_ns": 50125.00332668424, "command_buffers": 1 }, { "command_buffer_ns": 119875.01056864858, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 31.89453127561137, 30.181638976500835, 31.525391023023985, 30.55664183193585, 29.251956220832653, 30.007159693923313, 29.457682103384286, 29.61002610391006, 29.51106671389425 ], "command_buffer_throughput_us_p50": 30.007159693923313, "command_buffer_throughput_us_p90": 31.599219073541462, "command_buffer_latency_us": [ 71.0416934452951, 63.74996155500412, 49.875001423060894, 65.25003118440509, 52.91670095175505, 49.041700549423695, 49.541646149009466, 50.12500332668424, 119.87501056864858 ], "command_buffer_latency_us_p50": 52.91670095175505, "command_buffer_latency_us_p90": 80.8083568699658 }, "throughput": [ { "compute_ns": 3048001.0, "compute_span_ns": 4395667.0, "command_buffer_ns": 4264708.142727613, "calibration_cpu_ns": 5566708.0, "calibration_gpu_ticks": 5566708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3024037.0, "compute_span_ns": 4320833.0, "command_buffer_ns": 4186583.450064063, "calibration_cpu_ns": 5617042.0, "calibration_gpu_ticks": 5617042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3084456.0, "compute_span_ns": 4312709.0, "command_buffer_ns": 4161208.344157785, "calibration_cpu_ns": 5913959.0, "calibration_gpu_ticks": 5913959.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3036916.0, "compute_span_ns": 4340625.0, "command_buffer_ns": 4185833.2697302103, "calibration_cpu_ns": 5776375.0, "calibration_gpu_ticks": 5776375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3081956.0, "compute_span_ns": 4317000.0, "command_buffer_ns": 4155833.332333714, "calibration_cpu_ns": 5876625.0, "calibration_gpu_ticks": 5876625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3062420.0, "compute_span_ns": 4380250.0, "command_buffer_ns": 4228541.627526283, "calibration_cpu_ns": 5793292.0, "calibration_gpu_ticks": 5793292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2981495.0, "compute_span_ns": 3994416.0, "command_buffer_ns": 3832416.725344956, "calibration_cpu_ns": 5755708.0, "calibration_gpu_ticks": 5755708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3034212.0, "compute_span_ns": 4141583.0, "command_buffer_ns": 3864000.0275336206, "calibration_cpu_ns": 5903791.0, "calibration_gpu_ticks": 5903791.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3029711.0, "compute_span_ns": 4345125.0, "command_buffer_ns": 4180666.641332209, "calibration_cpu_ns": 5978000.0, "calibration_gpu_ticks": 5978000.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 47084.0, "compute_span_ns": 47084.0, "command_buffer_ns": 47125.03869086504, "calibration_cpu_ns": 506708.0, "calibration_gpu_ticks": 506708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49792.0, "compute_span_ns": 49792.0, "command_buffer_ns": 49833.382945507765, "calibration_cpu_ns": 641083.0, "calibration_gpu_ticks": 641083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49708.0, "compute_span_ns": 49708.0, "command_buffer_ns": 49666.676204651594, "calibration_cpu_ns": 550084.0, "calibration_gpu_ticks": 550084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47959.0, "compute_span_ns": 47959.0, "command_buffer_ns": 48000.01624971628, "calibration_cpu_ns": 544333.0, "calibration_gpu_ticks": 544333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49875.0, "compute_span_ns": 49875.0, "command_buffer_ns": 49875.001423060894, "calibration_cpu_ns": 652459.0, "calibration_gpu_ticks": 652459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72167.0, "compute_span_ns": 72167.0, "command_buffer_ns": 72208.34959298372, "calibration_cpu_ns": 1435125.0, "calibration_gpu_ticks": 1435125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 52708.0, "compute_span_ns": 52708.0, "command_buffer_ns": 52708.31752568483, "calibration_cpu_ns": 1448959.0, "calibration_gpu_ticks": 1448959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49084.0, "compute_span_ns": 49084.0, "command_buffer_ns": 49124.99571219087, "calibration_cpu_ns": 1101709.0, "calibration_gpu_ticks": 1101709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50959.0, "compute_span_ns": 50959.0, "command_buffer_ns": 50999.98088553548, "calibration_cpu_ns": 1202875.0, "calibration_gpu_ticks": 1202875.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.625015625, 47.250578125, 48.194625, 47.4518125, 48.1555625, 47.8503125, 46.585859375, 47.4095625, 47.339234375 ], "compute_throughput_us_p50": 47.4518125, "compute_throughput_us_p90": 48.163375, "command_buffer_throughput_us": [ 66.63606473011896, 65.41536640725099, 65.0188803774654, 65.40364483953454, 64.93489581771428, 66.07096293009818, 59.881511333514936, 60.37500043021282, 65.32291627081577 ], "command_buffer_throughput_us_p50": 65.32291627081577, "command_buffer_throughput_us_p90": 66.18398329010233, "compute_latency_us": [ 47.084, 49.792, 49.708, 47.959, 49.875, 72.167, 52.708, 49.084, 50.959 ], "compute_latency_us_p50": 49.792, "compute_latency_us_p90": 56.5998, "command_buffer_latency_us": [ 47.12503869086504, 49.833382945507765, 49.666676204651594, 48.00001624971628, 49.875001423060894, 72.20834959298372, 52.70831752568483, 49.12499571219087, 50.99998088553548 ], "command_buffer_latency_us_p50": 49.833382945507765, "command_buffer_latency_us_p90": 56.60832393914461, "counter_control_throughput_ratio": 2.1404068287104385, "counter_control_latency_ratio": 0.9425191012669646 }, "download_ms": 0.443917, "allocation_upload_ms": 1.77675, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.45640189840900125, "device_compute_slowdown": 0.4285597626012705, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.366459, "warmup_ms": 200.652084, "repetitions": 654, "throughput_us": [ 44.59403669724771, 44.17214678899083, 42.923483180428136, 46.17730733944954, 44.53911773700305, 44.3236498470948, 45.418642201834864, 44.000126911314986, 44.196801223241586 ], "latency_us": [ 2580.7090000000003, 984.0419999999999, 345.75, 348.125, 383.70799999999997, 301.70799999999997, 343.209, 363.042, 360.16700000000003 ], "throughput_us_p50": 44.3236498470948, "throughput_us_p90": 45.5703752293578, "latency_us_p50": 360.16700000000003, "latency_us_p90": 1303.3754000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1885333.1566788256, "command_buffers": 11 }, { "command_buffer_ns": 1945833.32631737, "command_buffers": 11 }, { "command_buffer_ns": 2020625.0483170152, "command_buffers": 11 }, { "command_buffer_ns": 1943749.8995102942, "command_buffers": 11 }, { "command_buffer_ns": 1905208.3953283727, "command_buffers": 11 }, { "command_buffer_ns": 1913083.3679810166, "command_buffers": 11 }, { "command_buffer_ns": 2030166.7391322553, "command_buffers": 11 }, { "command_buffer_ns": 1929291.698615998, "command_buffers": 11 }, { "command_buffer_ns": 2008999.8724870384, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 68499.99772384763, "command_buffers": 1 }, { "command_buffer_ns": 126958.35903286934, "command_buffers": 1 }, { "command_buffer_ns": 50916.62751510739, "command_buffers": 1 }, { "command_buffer_ns": 48833.375331014395, "command_buffers": 1 }, { "command_buffer_ns": 50416.62370786071, "command_buffers": 1 }, { "command_buffer_ns": 48374.99000132084, "command_buffers": 1 }, { "command_buffer_ns": 48000.01624971628, "command_buffers": 1 }, { "command_buffer_ns": 48124.9880976975, "command_buffers": 1 }, { "command_buffer_ns": 121124.96187910438, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.45833057310665, 30.403645723708905, 31.572266379953362, 30.371092179848347, 29.768881177005824, 29.891927624703385, 31.72135529894149, 30.14518279087497, 31.390623007609975 ], "command_buffer_throughput_us_p50": 30.371092179848347, "command_buffer_throughput_us_p90": 31.602084163750987, "command_buffer_latency_us": [ 68.49999772384763, 126.95835903286934, 50.91662751510739, 48.833375331014395, 50.41662370786071, 48.37499000132084, 48.00001624971628, 48.124988097697496, 121.12496187910438 ], "command_buffer_latency_us_p50": 50.41662370786071, "command_buffer_latency_us_p90": 122.29164130985737 }, "throughput": [ { "compute_ns": 3036163.0, "compute_span_ns": 4336875.0, "command_buffer_ns": 3905791.7310856283, "calibration_cpu_ns": 6098041.0, "calibration_gpu_ticks": 6098041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3087663.0, "compute_span_ns": 4443708.0, "command_buffer_ns": 3932833.3223238587, "calibration_cpu_ns": 5899791.0, "calibration_gpu_ticks": 5899791.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2989752.0, "compute_span_ns": 3926500.0, "command_buffer_ns": 3769333.241507411, "calibration_cpu_ns": 5833417.0, "calibration_gpu_ticks": 5833417.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3054710.0, "compute_span_ns": 4370083.0, "command_buffer_ns": 4219333.291985095, "calibration_cpu_ns": 5698167.0, "calibration_gpu_ticks": 5698167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3002752.0, "compute_span_ns": 4403167.0, "command_buffer_ns": 4217166.628222913, "calibration_cpu_ns": 5951584.0, "calibration_gpu_ticks": 5951584.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3045416.0, "compute_span_ns": 4536083.0, "command_buffer_ns": 4366750.072222203, "calibration_cpu_ns": 6014709.0, "calibration_gpu_ticks": 6014709.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3000540.0, "compute_span_ns": 4322042.0, "command_buffer_ns": 4182708.275038749, "calibration_cpu_ns": 5706125.0, "calibration_gpu_ticks": 5706125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3014915.0, "compute_span_ns": 4231708.0, "command_buffer_ns": 4058624.9670013785, "calibration_cpu_ns": 5854833.0, "calibration_gpu_ticks": 5854833.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2999250.0, "compute_span_ns": 4206917.0, "command_buffer_ns": 3884624.980855733, "calibration_cpu_ns": 6052833.0, "calibration_gpu_ticks": 6052833.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 48625.0, "compute_span_ns": 48625.0, "command_buffer_ns": 48625.050112605095, "calibration_cpu_ns": 1009000.0, "calibration_gpu_ticks": 1009000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50083.0, "compute_span_ns": 50083.0, "command_buffer_ns": 50041.708163917065, "calibration_cpu_ns": 769750.0, "calibration_gpu_ticks": 769750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 67708.0, "compute_span_ns": 67708.0, "command_buffer_ns": 67708.31532776356, "calibration_cpu_ns": 504083.0, "calibration_gpu_ticks": 504083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 119167.0, "compute_span_ns": 119167.0, "command_buffer_ns": 119208.35822820663, "calibration_cpu_ns": 1498209.0, "calibration_gpu_ticks": 1498209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 62542.0, "compute_span_ns": 62542.0, "command_buffer_ns": 62583.363614976406, "calibration_cpu_ns": 1434084.0, "calibration_gpu_ticks": 1434084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 51583.0, "compute_span_ns": 51583.0, "command_buffer_ns": 51583.33806321025, "calibration_cpu_ns": 1246584.0, "calibration_gpu_ticks": 1246584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 62917.0, "compute_span_ns": 62917.0, "command_buffer_ns": 62916.66068136692, "calibration_cpu_ns": 2082916.0, "calibration_gpu_ticks": 2082916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70167.0, "compute_span_ns": 70167.0, "command_buffer_ns": 70208.33436399698, "calibration_cpu_ns": 658584.0, "calibration_gpu_ticks": 658584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 51042.0, "compute_span_ns": 51042.0, "command_buffer_ns": 51083.334255963564, "calibration_cpu_ns": 1103167.0, "calibration_gpu_ticks": 1103167.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.440046875, 48.244734375, 46.714875, 47.72984375, 46.918, 47.584625, 46.8834375, 47.108046875, 46.86328125 ], "compute_throughput_us_p50": 47.108046875, "compute_throughput_us_p90": 47.832821875, "command_buffer_throughput_us": [ 61.02799579821294, 61.45052066131029, 58.8958318985533, 65.9270826872671, 65.89322856598301, 68.23046987847192, 65.35481679748045, 63.41601510939654, 60.69726532587083 ], "command_buffer_throughput_us_p50": 63.41601510939654, "command_buffer_throughput_us_p90": 66.38776012550807, "compute_latency_us": [ 48.625, 50.083, 67.708, 119.167, 62.542, 51.583, 62.917, 70.167, 51.042 ], "compute_latency_us_p50": 62.542, "compute_latency_us_p90": 79.96700000000001, "command_buffer_latency_us": [ 48.625050112605095, 50.041708163917065, 67.70831532776356, 119.20835822820663, 62.583363614976406, 51.58333806321025, 62.91666068136692, 70.20833436399698, 51.083334255963564 ], "command_buffer_latency_us_p50": 62.583363614976406, "command_buffer_latency_us_p90": 80.00833913683893, "counter_control_throughput_ratio": 2.071671904378965, "counter_control_latency_ratio": 1.24132397237895 }, "download_ms": 0.415417, "allocation_upload_ms": 0.521583, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-rncm118l/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "0d8a82e58cbe97c43d15ef559cf043ad575745e0d3fb0f4f40ed60b2caca21c0", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 22.367125, "capture_ms": 0.067333, "compile_ms": 53.919083, "allocation_upload_ms": 2.063958, "cold_call_ms": 5.710458, "warmup_ms": 200.187, "download_ms": 0.820584, "repetitions": 1237, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 20.0511649151, 19.9629482619, 20.1505658852, 20.391, 20.4376515764, 20.145109135, 20.2671107518, 20.1908852061, 20.2612845594 ], "latency_us": [ 307.5, 306.5, 244.791, 243.167, 248.583, 268.375, 230.958, 207.541, 228.542 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 837292, "compute_span_ns": 837292, "calibration_cpu_ns": 1301042, "calibration_gpu_ticks": 1301042, "compute_passes": 1, "command_buffer_ns": 837333.325762, "command_buffers": 2 }, { "compute_ns": 883792, "compute_span_ns": 883792, "calibration_cpu_ns": 1380458, "calibration_gpu_ticks": 1380458, "compute_passes": 1, "command_buffer_ns": 883833.33059, "command_buffers": 2 }, { "compute_ns": 1379458, "compute_span_ns": 1379458, "calibration_cpu_ns": 2542708, "calibration_gpu_ticks": 2542708, "compute_passes": 1, "command_buffer_ns": 1379541.7035, "command_buffers": 2 }, { "compute_ns": 856083, "compute_span_ns": 856083, "calibration_cpu_ns": 1596166, "calibration_gpu_ticks": 1596166, "compute_passes": 1, "command_buffer_ns": 856083.352119, "command_buffers": 2 }, { "compute_ns": 1327166, "compute_span_ns": 1327166, "calibration_cpu_ns": 2199667, "calibration_gpu_ticks": 2199667, "compute_passes": 1, "command_buffer_ns": 1327125.00636, "command_buffers": 2 }, { "compute_ns": 873667, "compute_span_ns": 873667, "calibration_cpu_ns": 1490333, "calibration_gpu_ticks": 1490333, "compute_passes": 1, "command_buffer_ns": 873749.959283, "command_buffers": 2 }, { "compute_ns": 1358583, "compute_span_ns": 1358583, "calibration_cpu_ns": 2630625, "calibration_gpu_ticks": 2630625, "compute_passes": 1, "command_buffer_ns": 1358625.01338, "command_buffers": 2 }, { "compute_ns": 873125, "compute_span_ns": 873125, "calibration_cpu_ns": 1423917, "calibration_gpu_ticks": 1423917, "compute_passes": 1, "command_buffer_ns": 873208.395205, "command_buffers": 2 }, { "compute_ns": 1123625, "compute_span_ns": 1123625, "calibration_cpu_ns": 2349875, "calibration_gpu_ticks": 2349875, "compute_passes": 1, "command_buffer_ns": 1123708.32358, "command_buffers": 2 } ], "latency": [ { "compute_ns": 23416, "compute_span_ns": 23416, "calibration_cpu_ns": 1064375, "calibration_gpu_ticks": 1064375, "compute_passes": 1, "command_buffer_ns": 23375.0324696, "command_buffers": 2 }, { "compute_ns": 18584, "compute_span_ns": 18584, "calibration_cpu_ns": 925125, "calibration_gpu_ticks": 925125, "compute_passes": 1, "command_buffer_ns": 18708.2914636, "command_buffers": 2 }, { "compute_ns": 87333, "compute_span_ns": 87333, "calibration_cpu_ns": 1451666, "calibration_gpu_ticks": 1451666, "compute_passes": 1, "command_buffer_ns": 87416.6144058, "command_buffers": 2 }, { "compute_ns": 15875, "compute_span_ns": 15875, "calibration_cpu_ns": 735375, "calibration_gpu_ticks": 735375, "compute_passes": 1, "command_buffer_ns": 15958.386939, "command_buffers": 2 }, { "compute_ns": 17541, "compute_span_ns": 17541, "calibration_cpu_ns": 1001250, "calibration_gpu_ticks": 1001250, "compute_passes": 1, "command_buffer_ns": 17625.046894, "command_buffers": 2 }, { "compute_ns": 18125, "compute_span_ns": 18125, "calibration_cpu_ns": 498000, "calibration_gpu_ticks": 498000, "compute_passes": 1, "command_buffer_ns": 18208.345864, "command_buffers": 2 }, { "compute_ns": 15792, "compute_span_ns": 15792, "calibration_cpu_ns": 513291, "calibration_gpu_ticks": 513291, "compute_passes": 1, "command_buffer_ns": 15874.9171533, "command_buffers": 2 }, { "compute_ns": 101041, "compute_span_ns": 101041, "calibration_cpu_ns": 1393667, "calibration_gpu_ticks": 1393667, "compute_passes": 1, "command_buffer_ns": 101083.307527, "command_buffers": 2 }, { "compute_ns": 18125, "compute_span_ns": 18125, "calibration_cpu_ns": 485750, "calibration_gpu_ticks": 485750, "compute_passes": 1, "command_buffer_ns": 18166.6109711, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 918374.978937, "command_buffers": 2 }, { "command_buffer_ns": 863083.347213, "command_buffers": 2 }, { "command_buffer_ns": 867541.646585, "command_buffers": 2 }, { "command_buffer_ns": 1094250.00846, "command_buffers": 2 }, { "command_buffer_ns": 880499.952473, "command_buffers": 2 }, { "command_buffer_ns": 1354750.01298, "command_buffers": 2 }, { "command_buffer_ns": 921041.646507, "command_buffers": 2 }, { "command_buffer_ns": 1149708.28872, "command_buffers": 2 }, { "command_buffer_ns": 850624.986924, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 15541.6200869, "command_buffers": 2 }, { "command_buffer_ns": 15666.6501425, "command_buffers": 2 }, { "command_buffer_ns": 18124.9924935, "command_buffers": 2 }, { "command_buffer_ns": 15666.6501425, "command_buffers": 2 }, { "command_buffer_ns": 42208.3539888, "command_buffers": 2 }, { "command_buffer_ns": 65583.3864585, "command_buffers": 2 }, { "command_buffer_ns": 20291.6562557, "command_buffers": 2 }, { "command_buffer_ns": 17749.9605343, "command_buffers": 2 }, { "command_buffer_ns": 15875.0335686, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 14.349609045890626, 13.485677300203124, 13.555338227890624, 17.0976563821875, 13.757811757390625, 21.1679689528125, 14.391275726671875, 17.964192011250002, 13.291015420687499 ], "command_buffer_throughput_us_p50": 14.349609045890626, "command_buffer_throughput_us_p90": 18.6049473995625, "command_buffer_latency_us": [ 15.5416200869, 15.6666501425, 18.124992493500002, 15.6666501425, 42.2083539888, 65.5833864585, 20.2916562557, 17.7499605343, 15.8750335686 ], "command_buffer_latency_us_p50": 17.7499605343, "command_buffer_latency_us_p90": 46.88336048274 }, "compute_throughput_us": [ 13.0826875, 13.80925, 21.55403125, 13.376296875, 20.73696875, 13.651046875, 21.227859375, 13.642578125, 17.556640625 ], "compute_throughput_us_p50": 13.80925, "compute_throughput_us_p90": 21.29309375, "command_buffer_throughput_us": [ 13.083333215031251, 13.809895790468751, 21.5553391171875, 13.376302376859377, 20.736328224375, 13.652343113796874, 21.2285158340625, 13.643881175078125, 17.5579425559375 ], "command_buffer_throughput_us_p50": 13.809895790468751, "command_buffer_throughput_us_p90": 21.2938804906875, "compute_latency_us": [ 23.416, 18.584, 87.333, 15.875, 17.541, 18.125, 15.792, 101.041, 18.125 ], "compute_latency_us_p50": 18.125, "compute_latency_us_p90": 90.0746, "command_buffer_latency_us": [ 23.375032469599997, 18.7082914636, 87.4166144058, 15.958386939, 17.625046894, 18.208345863999998, 15.8749171533, 101.083307527, 18.1666109711 ], "command_buffer_latency_us_p50": 18.208345863999998, "command_buffer_latency_us_p90": 90.14995303004001, "counter_control_throughput_ratio": 1.024041691273507, "counter_control_latency_ratio": 1.1443510272024005 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 20.1908852061, "throughput_us_p90": 20.400330315279998, "latency_us_p50": 244.791, "latency_us_p90": 306.7 }, "slowdown": 0.45553300045807066, "device_compute_slowdown": 0.29313993926860293, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-u3w4634t/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7f45b1ebfa78e24e3d477c2ad514fde5c49a3d7b63ed786757d9dc5d1979c8b1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.456084, "capture_ms": 0.074834, "compile_ms": 53.858458, "allocation_upload_ms": 6.973708, "cold_call_ms": 5.659125, "warmup_ms": 201.25525, "download_ms": 3.388709, "repetitions": 373, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 86.0979651475, 86.4674932976, 87.2794906166, 83.2625093834, 80.498772118, 86.6539329759, 86.6568364611, 86.1267882038, 84.3385817694 ], "latency_us": [ 1113.5, 383.291, 289.833, 290.25, 275.916, 274.459, 285.375, 267.833, 270.208 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3901292, "compute_span_ns": 3901292, "calibration_cpu_ns": 5033500, "calibration_gpu_ticks": 5033500, "compute_passes": 1, "command_buffer_ns": 3901416.66867, "command_buffers": 2 }, { "compute_ns": 4022625, "compute_span_ns": 4022625, "calibration_cpu_ns": 4991708, "calibration_gpu_ticks": 4991708, "compute_passes": 1, "command_buffer_ns": 4022624.98392, "command_buffers": 2 }, { "compute_ns": 4083333, "compute_span_ns": 4083333, "calibration_cpu_ns": 5244833, "calibration_gpu_ticks": 5244833, "compute_passes": 1, "command_buffer_ns": 4083458.33421, "command_buffers": 2 }, { "compute_ns": 4340833, "compute_span_ns": 4340833, "calibration_cpu_ns": 5531709, "calibration_gpu_ticks": 5531709, "compute_passes": 1, "command_buffer_ns": 4340916.6974, "command_buffers": 2 }, { "compute_ns": 4028459, "compute_span_ns": 4028459, "calibration_cpu_ns": 5155166, "calibration_gpu_ticks": 5155166, "compute_passes": 1, "command_buffer_ns": 4028541.67623, "command_buffers": 2 }, { "compute_ns": 4184292, "compute_span_ns": 4184292, "calibration_cpu_ns": 5380375, "calibration_gpu_ticks": 5380375, "compute_passes": 1, "command_buffer_ns": 4184374.93499, "command_buffers": 2 }, { "compute_ns": 3976875, "compute_span_ns": 3976875, "calibration_cpu_ns": 5115334, "calibration_gpu_ticks": 5115334, "compute_passes": 1, "command_buffer_ns": 3976916.66149, "command_buffers": 2 }, { "compute_ns": 4157417, "compute_span_ns": 4157417, "calibration_cpu_ns": 5374417, "calibration_gpu_ticks": 5374417, "compute_passes": 1, "command_buffer_ns": 4157458.3156, "command_buffers": 2 }, { "compute_ns": 3624500, "compute_span_ns": 3624500, "calibration_cpu_ns": 4587458, "calibration_gpu_ticks": 4587458, "compute_passes": 1, "command_buffer_ns": 3624500.0083, "command_buffers": 2 } ], "latency": [ { "compute_ns": 97750, "compute_span_ns": 97750, "calibration_cpu_ns": 1019917, "calibration_gpu_ticks": 1019917, "compute_passes": 1, "command_buffer_ns": 97833.3409876, "command_buffers": 2 }, { "compute_ns": 131500, "compute_span_ns": 131500, "calibration_cpu_ns": 1561875, "calibration_gpu_ticks": 1561875, "compute_passes": 1, "command_buffer_ns": 131583.365146, "command_buffers": 2 }, { "compute_ns": 112042, "compute_span_ns": 112042, "calibration_cpu_ns": 1449666, "calibration_gpu_ticks": 1449666, "compute_passes": 1, "command_buffer_ns": 112125.009764, "command_buffers": 2 }, { "compute_ns": 71625, "compute_span_ns": 71625, "calibration_cpu_ns": 649458, "calibration_gpu_ticks": 649458, "compute_passes": 1, "command_buffer_ns": 71624.9924153, "command_buffers": 2 }, { "compute_ns": 123584, "compute_span_ns": 123584, "calibration_cpu_ns": 1533458, "calibration_gpu_ticks": 1533458, "compute_passes": 1, "command_buffer_ns": 123708.392493, "command_buffers": 2 }, { "compute_ns": 99625, "compute_span_ns": 99625, "calibration_cpu_ns": 936667, "calibration_gpu_ticks": 936667, "compute_passes": 1, "command_buffer_ns": 99708.3261609, "command_buffers": 2 }, { "compute_ns": 116084, "compute_span_ns": 116084, "calibration_cpu_ns": 2428542, "calibration_gpu_ticks": 2428542, "compute_passes": 1, "command_buffer_ns": 116166.600492, "command_buffers": 2 }, { "compute_ns": 81250, "compute_span_ns": 81250, "calibration_cpu_ns": 606666, "calibration_gpu_ticks": 606666, "compute_passes": 1, "command_buffer_ns": 81291.6550785, "command_buffers": 2 }, { "compute_ns": 116375, "compute_span_ns": 116375, "calibration_cpu_ns": 635792, "calibration_gpu_ticks": 635792, "compute_passes": 1, "command_buffer_ns": 116416.602395, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3800458.33299, "command_buffers": 2 }, { "command_buffer_ns": 4115958.34885, "command_buffers": 2 }, { "command_buffer_ns": 4229000.01286, "command_buffers": 2 }, { "command_buffer_ns": 4047041.64248, "command_buffers": 2 }, { "command_buffer_ns": 4054041.69578, "command_buffers": 2 }, { "command_buffer_ns": 4116666.67787, "command_buffers": 2 }, { "command_buffer_ns": 3928958.32192, "command_buffers": 2 }, { "command_buffer_ns": 3956416.68001, "command_buffers": 2 }, { "command_buffer_ns": 3996583.34209, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 157166.679855, "command_buffers": 2 }, { "command_buffer_ns": 97875.0176728, "command_buffers": 2 }, { "command_buffer_ns": 64583.2624286, "command_buffers": 2 }, { "command_buffer_ns": 111458.357424, "command_buffers": 2 }, { "command_buffer_ns": 65999.9786876, "command_buffers": 2 }, { "command_buffer_ns": 83208.3169371, "command_buffers": 2 }, { "command_buffer_ns": 114874.972496, "command_buffers": 2 }, { "command_buffer_ns": 110499.968287, "command_buffers": 2 }, { "command_buffer_ns": 61458.3259448, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 59.38216145296875, 64.31184920078125, 66.0781252009375, 63.23502566375, 63.344401496562504, 64.32291684171875, 61.38997378, 61.819010625156245, 62.44661472015625 ], "command_buffer_throughput_us_p50": 63.23502566375, "command_buffer_throughput_us_p90": 64.6739585135625, "command_buffer_latency_us": [ 157.16667985499998, 97.8750176728, 64.5832624286, 111.458357424, 65.9999786876, 83.2083169371, 114.874972496, 110.499968287, 61.4583259448 ], "command_buffer_latency_us_p50": 97.8750176728, "command_buffer_latency_us_p90": 123.3333139678 }, "compute_throughput_us": [ 60.9576875, 62.853515625, 63.802078125, 67.825515625, 62.944671875, 65.3795625, 62.138671875, 64.959640625, 56.6328125 ], "compute_throughput_us_p50": 62.944671875, "compute_throughput_us_p90": 65.868753125, "command_buffer_throughput_us": [ 60.95963544796875, 62.85351537375, 63.80403647203125, 67.826823396875, 62.945963691093745, 65.38085835921875, 62.13932283578125, 64.96028618125, 56.6328126296875 ], "command_buffer_throughput_us_p50": 62.945963691093745, "command_buffer_throughput_us_p90": 65.87005136675, "compute_latency_us": [ 97.75, 131.5, 112.042, 71.625, 123.584, 99.625, 116.084, 81.25, 116.375 ], "compute_latency_us_p50": 112.042, "compute_latency_us_p90": 125.16720000000001, "command_buffer_latency_us": [ 97.8333409876, 131.583365146, 112.125009764, 71.6249924153, 123.708392493, 99.7083261609, 116.166600492, 81.2916550785, 116.416602395 ], "command_buffer_latency_us_p50": 112.125009764, "command_buffer_latency_us_p90": 125.2833870236, "counter_control_throughput_ratio": 1.0122063752375372, "counter_control_latency_ratio": 1.1982975960957476 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 86.1267882038, "throughput_us_p90": 86.7813672922, "latency_us_p50": 285.375, "latency_us_p90": 529.3328000000001 }, "torch": { "cold_call_ms": 46.657041, "warmup_ms": 201.901917, "repetitions": 171, "throughput_us": [ 187.6430292397661, 187.58625730994152, 187.87743859649123, 176.36062573099417, 185.37597660818713, 189.14790643274856, 188.14644444444446, 185.4712514619883, 181.1676432748538 ], "latency_us": [ 2069.166, 739.375, 549.458, 470.417, 430.417, 410.0, 394.667, 439.583, 418.5 ], "throughput_us_p50": 187.58625730994152, "throughput_us_p90": 188.3467368421053, "latency_us_p50": 439.583, "latency_us_p90": 1005.3332000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 10494166.635908186, "command_buffers": 11 }, { "command_buffer_ns": 10536291.694734246, "command_buffers": 11 }, { "command_buffer_ns": 10585250.158328563, "command_buffers": 11 }, { "command_buffer_ns": 10537166.672293097, "command_buffers": 11 }, { "command_buffer_ns": 10606958.414427936, "command_buffers": 11 }, { "command_buffer_ns": 10382458.33478868, "command_buffers": 11 }, { "command_buffer_ns": 10436249.955091625, "command_buffers": 11 }, { "command_buffer_ns": 10462749.807629734, "command_buffers": 11 }, { "command_buffer_ns": 10572624.974884093, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 167041.66773706675, "command_buffers": 1 }, { "command_buffer_ns": 140374.99204277992, "command_buffers": 1 }, { "command_buffer_ns": 189333.33922177553, "command_buffers": 1 }, { "command_buffer_ns": 236000.05079060793, "command_buffers": 1 }, { "command_buffer_ns": 152875.02901628613, "command_buffers": 1 }, { "command_buffer_ns": 148666.67333990335, "command_buffers": 1 }, { "command_buffer_ns": 137750.00115856528, "command_buffers": 1 }, { "command_buffer_ns": 160874.97351691127, "command_buffers": 1 }, { "command_buffer_ns": 143375.01488626003, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 163.9713536860654, 164.6295577302226, 165.3945337238838, 164.64322925457964, 165.7337252254365, 162.22591148107313, 163.06640554830665, 163.4804657442146, 165.19726523256395 ], "command_buffer_throughput_us_p50": 164.6295577302226, "command_buffer_throughput_us_p90": 165.46237202419434, "command_buffer_latency_us": [ 167.04166773706675, 140.37499204277992, 189.33333922177553, 236.00005079060793, 152.87502901628613, 148.66667333990335, 137.75000115856528, 160.87497351691127, 143.37501488626003 ], "command_buffer_latency_us_p50": 152.87502901628613, "command_buffer_latency_us_p90": 198.666681535542 }, "throughput": [ { "compute_ns": 11160246.0, "compute_span_ns": 12288625.0, "command_buffer_ns": 12106041.715014726, "calibration_cpu_ns": 13961041.0, "calibration_gpu_ticks": 13961041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11050546.0, "compute_span_ns": 12263542.0, "command_buffer_ns": 12078124.971594661, "calibration_cpu_ns": 13724167.0, "calibration_gpu_ticks": 13724167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11118912.0, "compute_span_ns": 12327625.0, "command_buffer_ns": 12158958.415966481, "calibration_cpu_ns": 13809750.0, "calibration_gpu_ticks": 13809750.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10961082.0, "compute_span_ns": 12259834.0, "command_buffer_ns": 12047125.084791332, "calibration_cpu_ns": 13837000.0, "calibration_gpu_ticks": 13837000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11041045.0, "compute_span_ns": 12271625.0, "command_buffer_ns": 12053708.254825324, "calibration_cpu_ns": 13887583.0, "calibration_gpu_ticks": 13887583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11062377.0, "compute_span_ns": 12280167.0, "command_buffer_ns": 12066458.410117775, "calibration_cpu_ns": 13794375.0, "calibration_gpu_ticks": 13794375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10999705.0, "compute_span_ns": 12261000.0, "command_buffer_ns": 12085750.058759004, "calibration_cpu_ns": 13831292.0, "calibration_gpu_ticks": 13831292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10927373.0, "compute_span_ns": 12183042.0, "command_buffer_ns": 11998916.801530868, "calibration_cpu_ns": 14081042.0, "calibration_gpu_ticks": 14081042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11092044.0, "compute_span_ns": 12288084.0, "command_buffer_ns": 12101041.618734598, "calibration_cpu_ns": 13626792.0, "calibration_gpu_ticks": 13626792.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 150583.0, "compute_span_ns": 150583.0, "command_buffer_ns": 150541.65851324797, "calibration_cpu_ns": 885625.0, "calibration_gpu_ticks": 885625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 139834.0, "compute_span_ns": 139834.0, "command_buffer_ns": 139874.98823553324, "calibration_cpu_ns": 642833.0, "calibration_gpu_ticks": 642833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 142375.0, "compute_span_ns": 142375.0, "command_buffer_ns": 142374.94906410575, "calibration_cpu_ns": 828166.0, "calibration_gpu_ticks": 828166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 139333.0, "compute_span_ns": 139333.0, "command_buffer_ns": 139333.3077430725, "calibration_cpu_ns": 633708.0, "calibration_gpu_ticks": 633708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 142625.0, "compute_span_ns": 142625.0, "command_buffer_ns": 142625.00917539, "calibration_cpu_ns": 833583.0, "calibration_gpu_ticks": 833583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 203291.0, "compute_span_ns": 203291.0, "command_buffer_ns": 203249.9760389328, "calibration_cpu_ns": 1494625.0, "calibration_gpu_ticks": 1494625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 140083.0, "compute_span_ns": 140083.0, "command_buffer_ns": 140041.6367687285, "calibration_cpu_ns": 645333.0, "calibration_gpu_ticks": 645333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 337584.0, "compute_span_ns": 337584.0, "command_buffer_ns": 337625.03881007433, "calibration_cpu_ns": 1428167.0, "calibration_gpu_ticks": 1428167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 139834.0, "compute_span_ns": 139834.0, "command_buffer_ns": 139874.98823553324, "calibration_cpu_ns": 627875.0, "calibration_gpu_ticks": 627875.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 174.37884375, 172.66478125, 173.733, 171.26690625, 172.516328125, 172.849640625, 171.870390625, 170.740203125, 173.3131875 ], "compute_throughput_us_p50": 172.66478125, "compute_throughput_us_p90": 173.86216875, "command_buffer_throughput_us": [ 189.1569017971051, 188.72070268116659, 189.98372524947627, 188.23632944986457, 188.33919148164568, 188.53841265809024, 188.83984466810944, 187.4830750239198, 189.0787752927281 ], "command_buffer_throughput_us_p50": 188.72070268116659, "command_buffer_throughput_us_p90": 189.32226648757933, "compute_latency_us": [ 150.583, 139.834, 142.375, 139.333, 142.625, 203.291, 140.083, 337.584, 139.834 ], "compute_latency_us_p50": 142.375, "compute_latency_us_p90": 230.14960000000002, "command_buffer_latency_us": [ 150.54165851324797, 139.87498823553324, 142.37494906410575, 139.3333077430725, 142.62500917539, 203.2499760389328, 140.0416367687285, 337.62503881007433, 139.87498823553324 ], "command_buffer_latency_us_p50": 142.37494906410575, "command_buffer_latency_us_p90": 230.12498859316113, "counter_control_throughput_ratio": 1.1468224914239005, "counter_control_latency_ratio": 0.9755883083708596 }, "download_ms": 0.687167, "allocation_upload_ms": 2.439667, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.4591316519604954, "device_compute_slowdown": 0.3645484123589911, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.719333, "warmup_ms": 200.986834, "repetitions": 167, "throughput_us": [ 188.9930119760479, 174.28517964071855, 191.57709580838323, 189.75399401197606, 181.47480239520957, 180.55813173652695, 188.49974850299404, 179.99625748502993, 181.36776646706585 ], "latency_us": [ 569.4169999999999, 807.958, 1946.2079999999999, 671.2080000000001, 676.2080000000001, 361.834, 423.917, 435.542, 399.625 ], "throughput_us_p50": 181.47480239520957, "throughput_us_p90": 190.1186143712575, "latency_us_p50": 569.4169999999999, "latency_us_p90": 1035.6080000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 10605083.429254591, "command_buffers": 11 }, { "command_buffer_ns": 10438541.64890945, "command_buffers": 11 }, { "command_buffer_ns": 10410041.722934693, "command_buffers": 11 }, { "command_buffer_ns": 10604333.3071284, "command_buffers": 11 }, { "command_buffer_ns": 10668500.035535544, "command_buffers": 11 }, { "command_buffer_ns": 10632999.998051673, "command_buffers": 11 }, { "command_buffer_ns": 10604916.722513735, "command_buffers": 11 }, { "command_buffer_ns": 10512208.33323896, "command_buffers": 11 }, { "command_buffer_ns": 10645958.303939551, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 216624.99057129025, "command_buffers": 1 }, { "command_buffer_ns": 316874.9972246587, "command_buffers": 1 }, { "command_buffer_ns": 159375.02030283213, "command_buffers": 1 }, { "command_buffer_ns": 156124.99555572867, "command_buffers": 1 }, { "command_buffer_ns": 141208.35112407804, "command_buffers": 1 }, { "command_buffer_ns": 137249.9973513186, "command_buffers": 1 }, { "command_buffer_ns": 315749.95955452323, "command_buffers": 1 }, { "command_buffer_ns": 350416.696164757, "command_buffers": 1 }, { "command_buffer_ns": 161500.00737980008, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 165.704428582103, 163.10221326421015, 162.65690192085458, 165.69270792388124, 166.69531305524288, 166.1406249695574, 165.7018237892771, 164.25325520685874, 166.3430984990555 ], "command_buffer_throughput_us_p50": 165.7018237892771, "command_buffer_throughput_us_p90": 166.41354141029296, "command_buffer_latency_us": [ 216.62499057129025, 316.8749972246587, 159.37502030283213, 156.12499555572867, 141.20835112407804, 137.2499973513186, 315.74995955452323, 350.416696164757, 161.50000737980008 ], "command_buffer_latency_us_p50": 161.50000737980008, "command_buffer_latency_us_p90": 323.5833370126784 }, "throughput": [ { "compute_ns": 10893285.0, "compute_span_ns": 12170916.0, "command_buffer_ns": 11952958.36078003, "calibration_cpu_ns": 14268042.0, "calibration_gpu_ticks": 14268042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10938753.0, "compute_span_ns": 12161125.0, "command_buffer_ns": 11949458.392336965, "calibration_cpu_ns": 13804583.0, "calibration_gpu_ticks": 13804583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11041004.0, "compute_span_ns": 12272709.0, "command_buffer_ns": 12078791.565727443, "calibration_cpu_ns": 14089041.0, "calibration_gpu_ticks": 14089041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10931628.0, "compute_span_ns": 12227542.0, "command_buffer_ns": 12073541.700374335, "calibration_cpu_ns": 13753584.0, "calibration_gpu_ticks": 13753584.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11077415.0, "compute_span_ns": 12307667.0, "command_buffer_ns": 12121749.925427139, "calibration_cpu_ns": 14598541.0, "calibration_gpu_ticks": 14598541.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10881084.0, "compute_span_ns": 12068708.0, "command_buffer_ns": 11890624.882653356, "calibration_cpu_ns": 13805666.0, "calibration_gpu_ticks": 13805666.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10969834.0, "compute_span_ns": 12298875.0, "command_buffer_ns": 12083000.037819147, "calibration_cpu_ns": 13577167.0, "calibration_gpu_ticks": 13577167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10923915.0, "compute_span_ns": 12152291.0, "command_buffer_ns": 11937999.981455505, "calibration_cpu_ns": 13641125.0, "calibration_gpu_ticks": 13641125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11049497.0, "compute_span_ns": 12280709.0, "command_buffer_ns": 12092208.431567997, "calibration_cpu_ns": 14057417.0, "calibration_gpu_ticks": 14057417.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 148334.0, "compute_span_ns": 148334.0, "command_buffer_ns": 148374.99475106597, "calibration_cpu_ns": 718584.0, "calibration_gpu_ticks": 718584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 140542.0, "compute_span_ns": 140542.0, "command_buffer_ns": 140583.31726118922, "calibration_cpu_ns": 779709.0, "calibration_gpu_ticks": 779709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 143750.0, "compute_span_ns": 143750.0, "command_buffer_ns": 143749.9886378646, "calibration_cpu_ns": 822584.0, "calibration_gpu_ticks": 822584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 297084.0, "compute_span_ns": 297084.0, "command_buffer_ns": 297124.9632537365, "calibration_cpu_ns": 1918500.0, "calibration_gpu_ticks": 1918500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 299500.0, "compute_span_ns": 299500.0, "command_buffer_ns": 299500.0104419887, "calibration_cpu_ns": 1849917.0, "calibration_gpu_ticks": 1849917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 161458.0, "compute_span_ns": 161458.0, "command_buffer_ns": 161458.38890224695, "calibration_cpu_ns": 863750.0, "calibration_gpu_ticks": 863750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 162750.0, "compute_span_ns": 162750.0, "command_buffer_ns": 162750.0168979168, "calibration_cpu_ns": 951500.0, "calibration_gpu_ticks": 951500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 145958.0, "compute_span_ns": 145958.0, "command_buffer_ns": 145958.32908526063, "calibration_cpu_ns": 740167.0, "calibration_gpu_ticks": 740167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 145208.0, "compute_span_ns": 145208.0, "command_buffer_ns": 145208.38158205152, "calibration_cpu_ns": 728708.0, "calibration_gpu_ticks": 728708.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 170.207578125, 170.918015625, 172.5156875, 170.8066875, 173.084609375, 170.0169375, 171.40365625, 170.686171875, 172.648390625 ], "compute_throughput_us_p50": 170.918015625, "compute_throughput_us_p90": 172.735634375, "command_buffer_throughput_us": [ 186.76497438718798, 186.71028738026507, 188.7311182144913, 188.64908906834899, 189.40234258479904, 185.79101379145868, 188.79687559092417, 186.53124971024226, 188.94075674324995 ], "command_buffer_throughput_us_p50": 188.64908906834899, "command_buffer_throughput_us_p90": 189.03307391155977, "compute_latency_us": [ 148.334, 140.542, 143.75, 297.084, 299.5, 161.458, 162.75, 145.958, 145.208 ], "compute_latency_us_p50": 148.334, "compute_latency_us_p90": 297.5672, "command_buffer_latency_us": [ 148.37499475106597, 140.58331726118922, 143.7499886378646, 297.1249632537365, 299.5000104419887, 161.45838890224695, 162.7500168979168, 145.95832908526063, 145.20838158205152 ], "command_buffer_latency_us_p50": 148.37499475106597, "command_buffer_latency_us_p90": 297.59997269138694, "counter_control_throughput_ratio": 1.1362187641234462, "counter_control_latency_ratio": 0.8991230646854678 }, "download_ms": 0.656167, "allocation_upload_ms": 2.753708, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-yts8etrn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "5e168786762f89acc561194a669a3725da4fe77bed5800b1d573576d096bb3fc", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.744958, "capture_ms": 0.075166, "compile_ms": 54.888708, "allocation_upload_ms": 6.949, "cold_call_ms": 8.062583, "warmup_ms": 203.753833, "download_ms": 3.339541, "repetitions": 380, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 91.8474789474, 90.5198447368, 92.8248894737, 95.4550447368, 91.1878289474, 91.2747815789, 91.3796052632, 98.3128289474, 92.0017552632 ], "latency_us": [ 354.125, 325.458, 291.75, 324.709, 291.875, 297.125, 532.583, 1976.334, 1041.208 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4519791, "compute_span_ns": 4519791, "calibration_cpu_ns": 5475791, "calibration_gpu_ticks": 5475791, "compute_passes": 1, "command_buffer_ns": 4519791.69156, "command_buffers": 2 }, { "compute_ns": 4379959, "compute_span_ns": 4379959, "calibration_cpu_ns": 5638125, "calibration_gpu_ticks": 5638125, "compute_passes": 1, "command_buffer_ns": 4380125.02855, "command_buffers": 2 }, { "compute_ns": 4586583, "compute_span_ns": 4586583, "calibration_cpu_ns": 5791750, "calibration_gpu_ticks": 5791750, "compute_passes": 1, "command_buffer_ns": 4586708.26629, "command_buffers": 2 }, { "compute_ns": 4467875, "compute_span_ns": 4467875, "calibration_cpu_ns": 5572083, "calibration_gpu_ticks": 5572083, "compute_passes": 1, "command_buffer_ns": 4467916.67491, "command_buffers": 2 }, { "compute_ns": 4174291, "compute_span_ns": 4174291, "calibration_cpu_ns": 5702709, "calibration_gpu_ticks": 5702709, "compute_passes": 1, "command_buffer_ns": 4174250.00342, "command_buffers": 2 }, { "compute_ns": 4410500, "compute_span_ns": 4410500, "calibration_cpu_ns": 5586084, "calibration_gpu_ticks": 5586084, "compute_passes": 1, "command_buffer_ns": 4410541.67459, "command_buffers": 2 }, { "compute_ns": 4663250, "compute_span_ns": 4663250, "calibration_cpu_ns": 5746333, "calibration_gpu_ticks": 5746333, "compute_passes": 1, "command_buffer_ns": 4663291.62009, "command_buffers": 2 }, { "compute_ns": 4441250, "compute_span_ns": 4441250, "calibration_cpu_ns": 5617792, "calibration_gpu_ticks": 5617792, "compute_passes": 1, "command_buffer_ns": 4441249.99922, "command_buffers": 2 }, { "compute_ns": 4130041, "compute_span_ns": 4130041, "calibration_cpu_ns": 5419917, "calibration_gpu_ticks": 5419917, "compute_passes": 1, "command_buffer_ns": 4130041.6342, "command_buffers": 2 } ], "latency": [ { "compute_ns": 77792, "compute_span_ns": 77792, "calibration_cpu_ns": 671625, "calibration_gpu_ticks": 671625, "compute_passes": 1, "command_buffer_ns": 77916.7166911, "command_buffers": 2 }, { "compute_ns": 64667, "compute_span_ns": 64667, "calibration_cpu_ns": 570875, "calibration_gpu_ticks": 570875, "compute_passes": 1, "command_buffer_ns": 64791.7040624, "command_buffers": 2 }, { "compute_ns": 109584, "compute_span_ns": 109584, "calibration_cpu_ns": 821417, "calibration_gpu_ticks": 821417, "compute_passes": 1, "command_buffer_ns": 109624.990728, "command_buffers": 2 }, { "compute_ns": 76917, "compute_span_ns": 76917, "calibration_cpu_ns": 622292, "calibration_gpu_ticks": 622292, "compute_passes": 1, "command_buffer_ns": 77083.3576098, "command_buffers": 2 }, { "compute_ns": 63791, "compute_span_ns": 63791, "calibration_cpu_ns": 522500, "calibration_gpu_ticks": 522500, "compute_passes": 1, "command_buffer_ns": 63750.0197627, "command_buffers": 2 }, { "compute_ns": 113333, "compute_span_ns": 113333, "calibration_cpu_ns": 819584, "calibration_gpu_ticks": 819584, "compute_passes": 1, "command_buffer_ns": 113416.695967, "command_buffers": 2 }, { "compute_ns": 64250, "compute_span_ns": 64250, "calibration_cpu_ns": 579250, "calibration_gpu_ticks": 579250, "compute_passes": 1, "command_buffer_ns": 64250.0235699, "command_buffers": 2 }, { "compute_ns": 63375, "compute_span_ns": 63375, "calibration_cpu_ns": 553375, "calibration_gpu_ticks": 553375, "compute_passes": 1, "command_buffer_ns": 63458.2829662, "command_buffers": 2 }, { "compute_ns": 116416, "compute_span_ns": 116416, "calibration_cpu_ns": 814000, "calibration_gpu_ticks": 814000, "compute_passes": 1, "command_buffer_ns": 116416.660603, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4356291.66896, "command_buffers": 2 }, { "command_buffer_ns": 4363666.69601, "command_buffers": 2 }, { "command_buffer_ns": 4355125.01281, "command_buffers": 2 }, { "command_buffer_ns": 4552291.648, "command_buffers": 2 }, { "command_buffer_ns": 4579041.67706, "command_buffers": 2 }, { "command_buffer_ns": 4463041.6669, "command_buffers": 2 }, { "command_buffer_ns": 4442583.3039, "command_buffers": 2 }, { "command_buffer_ns": 4449874.91937, "command_buffers": 2 }, { "command_buffer_ns": 4494208.31865, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 93458.336778, "command_buffers": 2 }, { "command_buffer_ns": 113791.669719, "command_buffers": 2 }, { "command_buffer_ns": 74041.6580811, "command_buffers": 2 }, { "command_buffer_ns": 112291.658297, "command_buffers": 2 }, { "command_buffer_ns": 78833.3709352, "command_buffers": 2 }, { "command_buffer_ns": 346624.990925, "command_buffers": 2 }, { "command_buffer_ns": 102958.2927, "command_buffers": 2 }, { "command_buffer_ns": 107958.330773, "command_buffers": 2 }, { "command_buffer_ns": 64124.9935143, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 68.06705732750001, 68.18229212515625, 68.04882832515625, 71.129557, 71.54752620406249, 69.7350260453125, 69.4153641234375, 69.52929561515626, 70.22200497890624 ], "command_buffer_throughput_us_p50": 69.52929561515626, "command_buffer_throughput_us_p90": 71.2131508408125, "command_buffer_latency_us": [ 93.458336778, 113.791669719, 74.0416580811, 112.291658297, 78.8333709352, 346.62499092499996, 102.9582927, 107.958330773, 64.1249935143 ], "command_buffer_latency_us_p50": 102.9582927, "command_buffer_latency_us_p90": 160.35833396020004 }, "compute_throughput_us": [ 70.621734375, 68.436859375, 71.665359375, 69.810546875, 65.223296875, 68.9140625, 72.86328125, 69.39453125, 64.531890625 ], "compute_throughput_us_p50": 69.39453125, "compute_throughput_us_p90": 71.90494375, "command_buffer_throughput_us": [ 70.621745180625, 68.43945357109375, 71.66731666078124, 69.81119804546874, 65.2226563034375, 68.91471366546875, 72.86393156390625, 69.3945312378125, 64.531900534375 ], "command_buffer_throughput_us_p50": 69.3945312378125, "command_buffer_throughput_us_p90": 71.90663964140624, "compute_latency_us": [ 77.792, 64.667, 109.584, 76.917, 63.791, 113.333, 64.25, 63.375, 116.416 ], "compute_latency_us_p50": 76.917, "compute_latency_us_p90": 113.9496, "command_buffer_latency_us": [ 77.91671669109999, 64.7917040624, 109.624990728, 77.0833576098, 63.7500197627, 113.41669596700001, 64.2500235699, 63.4582829662, 116.416660603 ], "command_buffer_latency_us_p50": 77.0833576098, "command_buffer_latency_us_p90": 114.0166888942, "counter_control_throughput_ratio": 0.9980617612166003, "counter_control_latency_ratio": 0.6864566680983762 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 91.8474789474, "throughput_us_p90": 96.02660157892001, "latency_us_p50": 325.458, "latency_us_p90": 1228.2332000000001 }, "slowdown": 0.5061169800718545, "device_compute_slowdown": 0.40601063027933804, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-2hns1lsn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "117627a7b60c4cee3b9bce9542bc5f280f025f1ad809d52b8d22cb3cdd24d0d1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.573416, "capture_ms": 0.085209, "compile_ms": 53.450542, "allocation_upload_ms": 6.961625, "cold_call_ms": 12.690541, "warmup_ms": 200.60225, "download_ms": 4.253542, "repetitions": 477, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 75.4270628931, 81.5457714885, 75.5539832285, 80.9418238994, 75.6175765199, 81.3245974843, 75.2530566038, 82.3698469602, 75.5038427673 ], "latency_us": [ 2247.417, 559.125, 386.084, 448.083, 276.542, 324.75, 288.583, 284.125, 286.167 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3747917, "compute_span_ns": 3747917, "calibration_cpu_ns": 4740291, "calibration_gpu_ticks": 4740291, "compute_passes": 1, "command_buffer_ns": 3747958.34068, "command_buffers": 2 }, { "compute_ns": 4007125, "compute_span_ns": 4007125, "calibration_cpu_ns": 5025459, "calibration_gpu_ticks": 5025459, "compute_passes": 1, "command_buffer_ns": 4007333.36573, "command_buffers": 2 }, { "compute_ns": 3888834, "compute_span_ns": 3888834, "calibration_cpu_ns": 5092625, "calibration_gpu_ticks": 5092625, "compute_passes": 1, "command_buffer_ns": 3888916.57349, "command_buffers": 2 }, { "compute_ns": 3837209, "compute_span_ns": 3837209, "calibration_cpu_ns": 5011667, "calibration_gpu_ticks": 5011667, "compute_passes": 1, "command_buffer_ns": 3837333.35184, "command_buffers": 2 }, { "compute_ns": 3746375, "compute_span_ns": 3746375, "calibration_cpu_ns": 4977791, "calibration_gpu_ticks": 4977791, "compute_passes": 1, "command_buffer_ns": 3746416.65258, "command_buffers": 2 }, { "compute_ns": 3674541, "compute_span_ns": 3674541, "calibration_cpu_ns": 5110417, "calibration_gpu_ticks": 5110417, "compute_passes": 1, "command_buffer_ns": 3674500.03978, "command_buffers": 2 }, { "compute_ns": 3743666, "compute_span_ns": 3743666, "calibration_cpu_ns": 5029041, "calibration_gpu_ticks": 5029041, "compute_passes": 1, "command_buffer_ns": 3743750.04321, "command_buffers": 2 }, { "compute_ns": 3651333, "compute_span_ns": 3651333, "calibration_cpu_ns": 5000541, "calibration_gpu_ticks": 5000541, "compute_passes": 1, "command_buffer_ns": 3651333.33253, "command_buffers": 2 }, { "compute_ns": 6113416, "compute_span_ns": 6113416, "calibration_cpu_ns": 7158541, "calibration_gpu_ticks": 7158541, "compute_passes": 1, "command_buffer_ns": 6113458.3666, "command_buffers": 2 } ], "latency": [ { "compute_ns": 106958, "compute_span_ns": 106958, "calibration_cpu_ns": 917875, "calibration_gpu_ticks": 917875, "compute_passes": 1, "command_buffer_ns": 107041.676529, "command_buffers": 2 }, { "compute_ns": 55583, "compute_span_ns": 55583, "calibration_cpu_ns": 657791, "calibration_gpu_ticks": 657791, "compute_passes": 1, "command_buffer_ns": 55541.691836, "command_buffers": 2 }, { "compute_ns": 99333, "compute_span_ns": 99333, "calibration_cpu_ns": 780834, "calibration_gpu_ticks": 780834, "compute_passes": 1, "command_buffer_ns": 99375.0290945, "command_buffers": 2 }, { "compute_ns": 67125, "compute_span_ns": 67125, "calibration_cpu_ns": 493167, "calibration_gpu_ticks": 493167, "compute_passes": 1, "command_buffer_ns": 67125.0163577, "command_buffers": 2 }, { "compute_ns": 58042, "compute_span_ns": 58042, "calibration_cpu_ns": 474792, "calibration_gpu_ticks": 474792, "compute_passes": 1, "command_buffer_ns": 58125.006035, "command_buffers": 2 }, { "compute_ns": 268708, "compute_span_ns": 268708, "calibration_cpu_ns": 1677167, "calibration_gpu_ticks": 1677167, "compute_passes": 1, "command_buffer_ns": 268750.009127, "command_buffers": 2 }, { "compute_ns": 56083, "compute_span_ns": 56083, "calibration_cpu_ns": 568750, "calibration_gpu_ticks": 568750, "compute_passes": 1, "command_buffer_ns": 56124.9325983, "command_buffers": 2 }, { "compute_ns": 99417, "compute_span_ns": 99417, "calibration_cpu_ns": 2650292, "calibration_gpu_ticks": 2650292, "compute_passes": 1, "command_buffer_ns": 99458.3242573, "command_buffers": 2 }, { "compute_ns": 177375, "compute_span_ns": 177375, "calibration_cpu_ns": 1531292, "calibration_gpu_ticks": 1531292, "compute_passes": 1, "command_buffer_ns": 177458.336111, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3777750.01107, "command_buffers": 2 }, { "command_buffer_ns": 3955458.34908, "command_buffers": 2 }, { "command_buffer_ns": 3882875.02574, "command_buffers": 2 }, { "command_buffer_ns": 3815375.03563, "command_buffers": 2 }, { "command_buffer_ns": 3918124.9449, "command_buffers": 2 }, { "command_buffer_ns": 3894000.02314, "command_buffers": 2 }, { "command_buffer_ns": 3814541.61834, "command_buffers": 2 }, { "command_buffer_ns": 6079250.01532, "command_buffers": 2 }, { "command_buffer_ns": 6087124.98797, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 58208.2429901, "command_buffers": 2 }, { "command_buffer_ns": 53666.5902473, "command_buffers": 2 }, { "command_buffer_ns": 100083.35812, "command_buffers": 2 }, { "command_buffer_ns": 99708.3261609, "command_buffers": 2 }, { "command_buffer_ns": 53750.0018254, "command_buffers": 2 }, { "command_buffer_ns": 98708.3185464, "command_buffers": 2 }, { "command_buffer_ns": 56958.3498873, "command_buffers": 2 }, { "command_buffer_ns": 54541.7424291, "command_buffers": 2 }, { "command_buffer_ns": 104375.008959, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 59.02734392296875, 61.804036704375, 60.669922277187496, 59.615234931718746, 61.2207022640625, 60.843750361562506, 59.602212786562504, 94.988281489375, 95.11132793703125 ], "command_buffer_throughput_us_p50": 60.843750361562506, "command_buffer_throughput_us_p90": 95.01289077890625, "command_buffer_latency_us": [ 58.2082429901, 53.6665902473, 100.08335812, 99.7083261609, 53.750001825400005, 98.7083185464, 56.9583498873, 54.5417424291, 104.375008959 ], "command_buffer_latency_us_p50": 58.2082429901, "command_buffer_latency_us_p90": 100.9416882878 }, "compute_throughput_us": [ 58.561203125, 62.611328125, 60.76303125, 59.956390625, 58.537109375, 57.414703125, 58.49478125, 57.052078125, 95.522125 ], "compute_throughput_us_p50": 58.561203125, "compute_throughput_us_p90": 69.1934875, "command_buffer_throughput_us": [ 58.561849073124996, 62.61458383953125, 60.76432146078125, 59.9583336225, 58.5377601965625, 57.414063121562506, 58.496094425156244, 57.05208332078125, 95.522786978125 ], "command_buffer_throughput_us_p50": 58.561849073124996, "command_buffer_throughput_us_p90": 69.19622446725, "compute_latency_us": [ 106.958, 55.583, 99.333, 67.125, 58.042, 268.708, 56.083, 99.417, 177.375 ], "compute_latency_us_p50": 99.333, "compute_latency_us_p90": 195.6416, "command_buffer_latency_us": [ 107.041676529, 55.541691836, 99.37502909450001, 67.1250163577, 58.125006035, 268.750009127, 56.1249325983, 99.4583242573, 177.45833611100002 ], "command_buffer_latency_us_p50": 99.37502909450001, "command_buffer_latency_us_p90": 195.71667071420003, "counter_control_throughput_ratio": 0.9921139116398118, "counter_control_latency_ratio": 1.0813954243910844 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 75.6175765199, "throughput_us_p90": 81.71058658284, "latency_us_p50": 324.75, "latency_us_p90": 896.7834000000003 }, "torch": { "cold_call_ms": 7.836417, "warmup_ms": 201.091625, "repetitions": 182, "throughput_us": [ 165.75686813186815, 163.7728901098901, 174.13804945054946, 172.15086813186815, 161.07051648351649, 171.9471153846154, 172.9020164835165, 163.4494065934066, 169.35828571428573 ], "latency_us": [ 576.4169999999999, 438.583, 441.29200000000003, 2396.292, 702.583, 430.375, 553.416, 482.208, 449.25 ], "throughput_us_p50": 169.35828571428573, "throughput_us_p90": 173.14922307692308, "latency_us_p50": 482.208, "latency_us_p90": 1041.3248000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 9757666.499353945, "command_buffers": 11 }, { "command_buffer_ns": 9820166.742429137, "command_buffers": 11 }, { "command_buffer_ns": 9801958.571188152, "command_buffers": 11 }, { "command_buffer_ns": 9897291.602101177, "command_buffers": 11 }, { "command_buffer_ns": 9672708.285506815, "command_buffers": 11 }, { "command_buffer_ns": 9831416.711676866, "command_buffers": 11 }, { "command_buffer_ns": 9788958.413992077, "command_buffers": 11 }, { "command_buffer_ns": 9887499.909382313, "command_buffers": 11 }, { "command_buffer_ns": 9827791.713178158, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 150333.33329483867, "command_buffers": 1 }, { "command_buffer_ns": 264499.9767653644, "command_buffers": 1 }, { "command_buffer_ns": 134125.00265985727, "command_buffers": 1 }, { "command_buffer_ns": 313291.61720350385, "command_buffers": 1 }, { "command_buffer_ns": 145583.35533365607, "command_buffers": 1 }, { "command_buffer_ns": 140624.99394640326, "command_buffers": 1 }, { "command_buffer_ns": 123250.00716373324, "command_buffers": 1 }, { "command_buffer_ns": 125249.96418505907, "command_buffers": 1 }, { "command_buffer_ns": 128791.66752099991, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 152.4635390524054, 153.44010535045527, 153.15560267481487, 154.6451812828309, 151.13606696104398, 153.61588611995103, 152.9524752186262, 154.49218608409865, 153.55924551840872 ], "command_buffer_throughput_us_p50": 153.44010535045527, "command_buffer_throughput_us_p90": 154.5227851238451, "command_buffer_latency_us": [ 150.33333329483867, 264.4999767653644, 134.12500265985727, 313.29161720350385, 145.58335533365607, 140.62499394640326, 123.25000716373324, 125.24996418505907, 128.7916675209999 ], "command_buffer_latency_us_p50": 140.62499394640326, "command_buffer_latency_us_p90": 274.2583048529923 }, "throughput": [ { "compute_ns": 10161083.0, "compute_span_ns": 11378375.0, "command_buffer_ns": 11192875.099368393, "calibration_cpu_ns": 13124125.0, "calibration_gpu_ticks": 13124125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9971376.0, "compute_span_ns": 11245875.0, "command_buffer_ns": 11029625.07866323, "calibration_cpu_ns": 12827000.0, "calibration_gpu_ticks": 12827000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9993001.0, "compute_span_ns": 11306708.0, "command_buffer_ns": 11114708.380773664, "calibration_cpu_ns": 13088000.0, "calibration_gpu_ticks": 13088000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10086043.0, "compute_span_ns": 11300375.0, "command_buffer_ns": 11120499.984826893, "calibration_cpu_ns": 12861750.0, "calibration_gpu_ticks": 12861750.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10085872.0, "compute_span_ns": 11347416.0, "command_buffer_ns": 11150708.36385712, "calibration_cpu_ns": 12846458.0, "calibration_gpu_ticks": 12846458.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10077209.0, "compute_span_ns": 11250167.0, "command_buffer_ns": 11064500.024076551, "calibration_cpu_ns": 12590916.0, "calibration_gpu_ticks": 12590916.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10263795.0, "compute_span_ns": 11427292.0, "command_buffer_ns": 11250791.721977293, "calibration_cpu_ns": 12698250.0, "calibration_gpu_ticks": 12698250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9994502.0, "compute_span_ns": 11103750.0, "command_buffer_ns": 10929708.369076252, "calibration_cpu_ns": 12608083.0, "calibration_gpu_ticks": 12608083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10043420.0, "compute_span_ns": 11239041.0, "command_buffer_ns": 11060124.84524399, "calibration_cpu_ns": 12939000.0, "calibration_gpu_ticks": 12939000.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 125292.0, "compute_span_ns": 125292.0, "command_buffer_ns": 125291.64087027311, "calibration_cpu_ns": 855917.0, "calibration_gpu_ticks": 855917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125750.0, "compute_span_ns": 125750.0, "command_buffer_ns": 125750.02619996667, "calibration_cpu_ns": 651584.0, "calibration_gpu_ticks": 651584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 128709.0, "compute_span_ns": 128709.0, "command_buffer_ns": 128749.99083578587, "calibration_cpu_ns": 639084.0, "calibration_gpu_ticks": 639084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125709.0, "compute_span_ns": 125709.0, "command_buffer_ns": 125750.02619996667, "calibration_cpu_ns": 621792.0, "calibration_gpu_ticks": 621792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 129791.0, "compute_span_ns": 129791.0, "command_buffer_ns": 129749.99845027924, "calibration_cpu_ns": 821708.0, "calibration_gpu_ticks": 821708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271833.0, "compute_span_ns": 271833.0, "command_buffer_ns": 271791.7086556554, "calibration_cpu_ns": 1819708.0, "calibration_gpu_ticks": 1819708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 159375.0, "compute_span_ns": 159375.0, "command_buffer_ns": 159374.9620951712, "calibration_cpu_ns": 1657208.0, "calibration_gpu_ticks": 1657208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137125.0, "compute_span_ns": 137125.0, "command_buffer_ns": 137124.96729567647, "calibration_cpu_ns": 914084.0, "calibration_gpu_ticks": 914084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 313958.0, "compute_span_ns": 313958.0, "command_buffer_ns": 313958.3277516067, "calibration_cpu_ns": 1559542.0, "calibration_gpu_ticks": 1559542.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 158.766921875, 155.80275, 156.140640625, 157.594421875, 157.59175, 157.456390625, 160.371796875, 156.16409375, 156.9284375 ], "compute_throughput_us_p50": 157.456390625, "compute_throughput_us_p90": 159.087896875, "command_buffer_throughput_us": [ 174.88867342763115, 172.33789185411297, 173.6673184495885, 173.7578122629202, 174.2298181852675, 172.8828128761961, 175.7936206558952, 170.77669326681644, 172.81445070693735 ], "command_buffer_throughput_us_p50": 173.6673184495885, "command_buffer_throughput_us_p90": 175.06966287328396, "compute_latency_us": [ 125.292, 125.75, 128.709, 125.709, 129.791, 271.833, 159.375, 137.125, 313.958 ], "compute_latency_us_p50": 129.791, "compute_latency_us_p90": 280.25800000000004, "command_buffer_latency_us": [ 125.29164087027311, 125.75002619996667, 128.74999083578587, 125.75002619996667, 129.74999845027924, 271.7917086556554, 159.3749620951712, 137.12496729567647, 313.9583277516067 ], "command_buffer_latency_us_p50": 129.74999845027924, "command_buffer_latency_us_p90": 280.22503247484565, "counter_control_throughput_ratio": 1.1254227491888469, "counter_control_latency_ratio": 0.9599253553216882 }, "download_ms": 0.890708, "allocation_upload_ms": 17.267334, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.44649469732747477, "device_compute_slowdown": 0.3719201417773512, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.56175, "warmup_ms": 200.174042, "repetitions": 165, "throughput_us": [ 165.99898787878786, 183.69545454545454, 164.73333333333332, 169.83737575757576, 172.15075757575758, 164.7300484848485, 173.41287878787878, 168.14646666666667, 165.61767272727275 ], "latency_us": [ 592.875, 2395.6659999999997, 801.041, 422.29100000000005, 525.375, 422.084, 484.125, 408.45799999999997, 406.208 ], "throughput_us_p50": 168.14646666666667, "throughput_us_p90": 175.46939393939394, "latency_us_p50": 484.125, "latency_us_p90": 1119.9660000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 9737749.93326515, "command_buffers": 11 }, { "command_buffer_ns": 9754208.498634398, "command_buffers": 11 }, { "command_buffer_ns": 9818499.96605888, "command_buffers": 11 }, { "command_buffer_ns": 9876625.030301511, "command_buffers": 11 }, { "command_buffer_ns": 9699749.993160367, "command_buffers": 11 }, { "command_buffer_ns": 9830291.67400673, "command_buffers": 11 }, { "command_buffer_ns": 9903416.619636118, "command_buffers": 11 }, { "command_buffer_ns": 9849666.617810726, "command_buffers": 11 }, { "command_buffer_ns": 9902708.290610462, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 187625.00258162618, "command_buffers": 1 }, { "command_buffer_ns": 282208.37702974677, "command_buffers": 1 }, { "command_buffer_ns": 141958.35683494806, "command_buffers": 1 }, { "command_buffer_ns": 130874.97791275382, "command_buffers": 1 }, { "command_buffer_ns": 128458.31224694848, "command_buffers": 1 }, { "command_buffer_ns": 131666.6603088379, "command_buffers": 1 }, { "command_buffer_ns": 283291.6798070073, "command_buffers": 1 }, { "command_buffer_ns": 313666.6491627693, "command_buffers": 1 }, { "command_buffer_ns": 138791.62725061178, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 152.15234270726796, 152.40950779116247, 153.41406196967, 154.3222660984611, 151.55859364313073, 153.59830740635516, 154.74088468181435, 153.9010409032926, 154.72981704078848 ], "command_buffer_throughput_us_p50": 153.59830740635516, "command_buffer_throughput_us_p90": 154.73203056899365, "command_buffer_latency_us": [ 187.62500258162618, 282.20837702974677, 141.95835683494806, 130.87497791275382, 128.45831224694848, 131.6666603088379, 283.2916798070073, 313.6666491627693, 138.79162725061178 ], "command_buffer_latency_us_p50": 141.95835683494806, "command_buffer_latency_us_p90": 289.3666736781597 }, "throughput": [ { "compute_ns": 10063496.0, "compute_span_ns": 11395584.0, "command_buffer_ns": 11200624.925550073, "calibration_cpu_ns": 13139500.0, "calibration_gpu_ticks": 13139500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10019501.0, "compute_span_ns": 11229125.0, "command_buffer_ns": 11054916.714783758, "calibration_cpu_ns": 13217250.0, "calibration_gpu_ticks": 13217250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10079081.0, "compute_span_ns": 11226208.0, "command_buffer_ns": 11017958.2843557, "calibration_cpu_ns": 12804084.0, "calibration_gpu_ticks": 12804084.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10115495.0, "compute_span_ns": 11244166.0, "command_buffer_ns": 11045916.588045657, "calibration_cpu_ns": 12771375.0, "calibration_gpu_ticks": 12771375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10075545.0, "compute_span_ns": 11350459.0, "command_buffer_ns": 11146500.008180737, "calibration_cpu_ns": 12784833.0, "calibration_gpu_ticks": 12784833.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10154751.0, "compute_span_ns": 11346750.0, "command_buffer_ns": 11162708.39702338, "calibration_cpu_ns": 12955500.0, "calibration_gpu_ticks": 12955500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10391995.0, "compute_span_ns": 11566791.0, "command_buffer_ns": 11395541.66002199, "calibration_cpu_ns": 13288917.0, "calibration_gpu_ticks": 13288917.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10017413.0, "compute_span_ns": 11190125.0, "command_buffer_ns": 11034916.56249389, "calibration_cpu_ns": 12355583.0, "calibration_gpu_ticks": 12355583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10265796.0, "compute_span_ns": 11428042.0, "command_buffer_ns": 11269250.011537224, "calibration_cpu_ns": 12584166.0, "calibration_gpu_ticks": 12584166.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 137667.0, "compute_span_ns": 137667.0, "command_buffer_ns": 137708.32447335124, "calibration_cpu_ns": 946542.0, "calibration_gpu_ticks": 946542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 127667.0, "compute_span_ns": 127667.0, "command_buffer_ns": 127666.68805852532, "calibration_cpu_ns": 652875.0, "calibration_gpu_ticks": 652875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 124084.0, "compute_span_ns": 124084.0, "command_buffer_ns": 124125.0429302454, "calibration_cpu_ns": 623791.0, "calibration_gpu_ticks": 623791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 191916.0, "compute_span_ns": 191916.0, "command_buffer_ns": 191875.034943223, "calibration_cpu_ns": 1819833.0, "calibration_gpu_ticks": 1819833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 190625.0, "compute_span_ns": 190625.0, "command_buffer_ns": 190625.0254251063, "calibration_cpu_ns": 1687041.0, "calibration_gpu_ticks": 1687041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136666.0, "compute_span_ns": 136666.0, "command_buffer_ns": 136624.96348842978, "calibration_cpu_ns": 871666.0, "calibration_gpu_ticks": 871666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 143333.0, "compute_span_ns": 143333.0, "command_buffer_ns": 143291.66151583195, "calibration_cpu_ns": 969542.0, "calibration_gpu_ticks": 969542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 123791.0, "compute_span_ns": 123791.0, "command_buffer_ns": 123750.01097097993, "calibration_cpu_ns": 673833.0, "calibration_gpu_ticks": 673833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 120667.0, "compute_span_ns": 120667.0, "command_buffer_ns": 120708.31144228578, "calibration_cpu_ns": 623416.0, "calibration_gpu_ticks": 623416.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 157.242125, 156.554703125, 157.485640625, 158.054609375, 157.430390625, 158.667984375, 162.374921875, 156.522078125, 160.4030625 ], "compute_throughput_us_p50": 157.485640625, "compute_throughput_us_p90": 160.797434375, "command_buffer_throughput_us": [ 175.0097644617199, 172.73307366849622, 172.1555981930578, 172.59244668821339, 174.16406262782402, 174.4173187034903, 178.0553384378436, 172.42057128896704, 176.08203143026913 ], "command_buffer_throughput_us_p50": 174.16406262782402, "command_buffer_throughput_us_p90": 176.47669283178402, "compute_latency_us": [ 137.667, 127.667, 124.084, 191.916, 190.625, 136.666, 143.333, 123.791, 120.667 ], "compute_latency_us_p50": 136.666, "compute_latency_us_p90": 190.8832, "command_buffer_latency_us": [ 137.70832447335124, 127.66668805852532, 124.1250429302454, 191.875034943223, 190.6250254251063, 136.62496348842978, 143.29166151583195, 123.75001097097993, 120.70831144228578 ], "command_buffer_latency_us_p50": 136.62496348842978, "command_buffer_latency_us_p90": 190.87502732872963, "counter_control_throughput_ratio": 1.1355419317353348, "counter_control_latency_ratio": 0.869708885423805 }, "download_ms": 0.666417, "allocation_upload_ms": 0.815625, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-3krqg6bl/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "8bb581805080afef9f47b3147742261e73f0bbc7f1d60afd69631b609a5e3f17", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.152667, "capture_ms": 0.065417, "compile_ms": 53.664625, "allocation_upload_ms": 6.232625, "cold_call_ms": 5.310833, "warmup_ms": 200.55125, "download_ms": 3.276958, "repetitions": 417, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 76.5001990408, 72.9216618705, 73.2910671463, 76.5043980815, 76.6988417266, 71.7919664269, 74.1902470024, 76.5946235012, 76.3673069544 ], "latency_us": [ 406.917, 386.666, 335.875, 337.167, 285.667, 295.667, 286.75, 292.125, 321.25 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3557833, "compute_span_ns": 3557833, "calibration_cpu_ns": 4135583, "calibration_gpu_ticks": 4135583, "compute_passes": 1, "command_buffer_ns": 3557916.61423, "command_buffers": 2 }, { "compute_ns": 3660166, "compute_span_ns": 3660166, "calibration_cpu_ns": 4774000, "calibration_gpu_ticks": 4774000, "compute_passes": 1, "command_buffer_ns": 3660208.31279, "command_buffers": 2 }, { "compute_ns": 3693917, "compute_span_ns": 3693917, "calibration_cpu_ns": 4810208, "calibration_gpu_ticks": 4810208, "compute_passes": 1, "command_buffer_ns": 3693958.33695, "command_buffers": 2 }, { "compute_ns": 3463083, "compute_span_ns": 3463083, "calibration_cpu_ns": 4711167, "calibration_gpu_ticks": 4711167, "compute_passes": 1, "command_buffer_ns": 3463166.70766, "command_buffers": 2 }, { "compute_ns": 3783834, "compute_span_ns": 3783834, "calibration_cpu_ns": 4882542, "calibration_gpu_ticks": 4882542, "compute_passes": 1, "command_buffer_ns": 3783958.32377, "command_buffers": 2 }, { "compute_ns": 3625958, "compute_span_ns": 3625958, "calibration_cpu_ns": 4750375, "calibration_gpu_ticks": 4750375, "compute_passes": 1, "command_buffer_ns": 3625958.34304, "command_buffers": 2 }, { "compute_ns": 3443833, "compute_span_ns": 3443833, "calibration_cpu_ns": 4966542, "calibration_gpu_ticks": 4966542, "compute_passes": 1, "command_buffer_ns": 3443916.7357, "command_buffers": 2 }, { "compute_ns": 3728209, "compute_span_ns": 3728209, "calibration_cpu_ns": 5030584, "calibration_gpu_ticks": 5030584, "compute_passes": 1, "command_buffer_ns": 3728333.39497, "command_buffers": 2 }, { "compute_ns": 6032042, "compute_span_ns": 6032042, "calibration_cpu_ns": 7237667, "calibration_gpu_ticks": 7237667, "compute_passes": 1, "command_buffer_ns": 6032166.65331, "command_buffers": 2 } ], "latency": [ { "compute_ns": 364167, "compute_span_ns": 364167, "calibration_cpu_ns": 1898709, "calibration_gpu_ticks": 1898709, "compute_passes": 1, "command_buffer_ns": 364208.302926, "command_buffers": 2 }, { "compute_ns": 101792, "compute_span_ns": 101792, "calibration_cpu_ns": 596958, "calibration_gpu_ticks": 596958, "compute_passes": 1, "command_buffer_ns": 101791.636553, "command_buffers": 2 }, { "compute_ns": 92208, "compute_span_ns": 92208, "calibration_cpu_ns": 1048000, "calibration_gpu_ticks": 1048000, "compute_passes": 1, "command_buffer_ns": 92166.6505747, "command_buffers": 2 }, { "compute_ns": 55250, "compute_span_ns": 55250, "calibration_cpu_ns": 559500, "calibration_gpu_ticks": 559500, "compute_passes": 1, "command_buffer_ns": 55333.3084099, "command_buffers": 2 }, { "compute_ns": 96333, "compute_span_ns": 96333, "calibration_cpu_ns": 701209, "calibration_gpu_ticks": 701209, "compute_passes": 1, "command_buffer_ns": 96374.9480434, "command_buffers": 2 }, { "compute_ns": 64833, "compute_span_ns": 64833, "calibration_cpu_ns": 595667, "calibration_gpu_ticks": 595667, "compute_passes": 1, "command_buffer_ns": 64874.9410175, "command_buffers": 2 }, { "compute_ns": 96208, "compute_span_ns": 96208, "calibration_cpu_ns": 948625, "calibration_gpu_ticks": 948625, "compute_passes": 1, "command_buffer_ns": 96291.7110883, "command_buffers": 2 }, { "compute_ns": 62291, "compute_span_ns": 62291, "calibration_cpu_ns": 485875, "calibration_gpu_ticks": 485875, "compute_passes": 1, "command_buffer_ns": 62250.0083409, "command_buffers": 2 }, { "compute_ns": 97500, "compute_span_ns": 97500, "calibration_cpu_ns": 3102708, "calibration_gpu_ticks": 3102708, "compute_passes": 1, "command_buffer_ns": 97541.6623987, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4556625.03373, "command_buffers": 2 }, { "command_buffer_ns": 3630666.64431, "command_buffers": 2 }, { "command_buffer_ns": 3748041.63584, "command_buffers": 2 }, { "command_buffer_ns": 3767875.02319, "command_buffers": 2 }, { "command_buffer_ns": 3647708.33403, "command_buffers": 2 }, { "command_buffer_ns": 3609125.03675, "command_buffers": 2 }, { "command_buffer_ns": 3715416.64936, "command_buffers": 2 }, { "command_buffer_ns": 3422708.30879, "command_buffers": 2 }, { "command_buffer_ns": 3402124.97395, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 70500.0129528, "command_buffers": 2 }, { "command_buffer_ns": 52791.6126885, "command_buffers": 2 }, { "command_buffer_ns": 54583.302699, "command_buffers": 2 }, { "command_buffer_ns": 53416.704759, "command_buffers": 2 }, { "command_buffer_ns": 100750.010461, "command_buffers": 2 }, { "command_buffer_ns": 101500.016171, "command_buffers": 2 }, { "command_buffer_ns": 52999.9961145, "command_buffers": 2 }, { "command_buffer_ns": 52625.0223629, "command_buffers": 2 }, { "command_buffer_ns": 98374.9632724, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 71.19726615203125, 56.72916631734375, 58.563150560000004, 58.873047237343755, 56.99544271921875, 56.39257869921875, 58.05338514625, 53.47981732484375, 53.15820271796875 ], "command_buffer_throughput_us_p50": 56.99544271921875, "command_buffer_throughput_us_p90": 61.337891020281255, "command_buffer_latency_us": [ 70.50001295279999, 52.791612688499995, 54.583302699, 53.416704759, 100.750010461, 101.500016171, 52.9999961145, 52.625022362900005, 98.3749632724 ], "command_buffer_latency_us_p50": 54.583302699, "command_buffer_latency_us_p90": 100.900011603 }, "compute_throughput_us": [ 55.591140625, 57.19009375, 57.717453125, 54.110671875, 59.12240625, 56.65559375, 53.809890625, 58.253265625, 94.25065625 ], "compute_throughput_us_p50": 57.19009375, "compute_throughput_us_p90": 66.14805625000001, "command_buffer_throughput_us": [ 55.59244709734375, 57.19075488734375, 57.71809901484375, 54.111979807187495, 59.124348808906255, 56.655599110000004, 53.8111989953125, 58.25520929640625, 94.25260395796874 ], "command_buffer_throughput_us_p50": 57.19075488734375, "command_buffer_throughput_us_p90": 66.14999983871876, "compute_latency_us": [ 364.167, 101.792, 92.208, 55.25, 96.333, 64.833, 96.208, 62.291, 97.5 ], "compute_latency_us_p50": 96.208, "compute_latency_us_p90": 154.26700000000005, "command_buffer_latency_us": [ 364.208302926, 101.791636553, 92.1666505747, 55.3333084099, 96.3749480434, 64.8749410175, 96.2917110883, 62.2500083409, 97.54166239870001 ], "command_buffer_latency_us_p50": 96.2917110883, "command_buffer_latency_us_p90": 154.27496982760005, "counter_control_throughput_ratio": 1.0046640961780473, "counter_control_latency_ratio": 1.182897518059498 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 76.3673069544, "throughput_us_p90": 76.61546714628, "latency_us_p50": 321.25, "latency_us_p90": 390.7162 }, "slowdown": 0.4541713451867558, "device_compute_slowdown": 0.3631448144925118, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-31vvscl6/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "54d99166e42515c7bfff1cac173ad46750fa91f9c92e668c07d69318fdc050bf", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.166292, "capture_ms": 0.075417, "compile_ms": 56.895292, "allocation_upload_ms": 1.60625, "cold_call_ms": 1.991291, "warmup_ms": 200.131625, "download_ms": 0.309084, "repetitions": 3641, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 5.57766849767, 6.41448091184, 6.15398709146, 5.47525872013, 6.3300032958, 5.63788547102, 5.84887393573, 5.85574018127, 5.56058308157 ], "latency_us": [ 244.625, 217.25, 194.375, 195.458, 202.667, 192.041, 191.417, 200.917, 199.833 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 56, "normalized_kernel_cost": 56, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 536584, "compute_span_ns": 536584, "calibration_cpu_ns": 2219541, "calibration_gpu_ticks": 2219541, "compute_passes": 1, "command_buffer_ns": 536666.71738, "command_buffers": 2 }, { "compute_ns": 291458, "compute_span_ns": 291458, "calibration_cpu_ns": 769500, "calibration_gpu_ticks": 769500, "compute_passes": 1, "command_buffer_ns": 291500.007734, "command_buffers": 2 }, { "compute_ns": 265000, "compute_span_ns": 265000, "calibration_cpu_ns": 698875, "calibration_gpu_ticks": 698875, "compute_passes": 1, "command_buffer_ns": 265041.715465, "command_buffers": 2 }, { "compute_ns": 247209, "compute_span_ns": 247209, "calibration_cpu_ns": 609250, "calibration_gpu_ticks": 609250, "compute_passes": 1, "command_buffer_ns": 247333.373409, "command_buffers": 2 }, { "compute_ns": 273417, "compute_span_ns": 273417, "calibration_cpu_ns": 781833, "calibration_gpu_ticks": 781833, "compute_passes": 1, "command_buffer_ns": 273541.663773, "command_buffers": 2 }, { "compute_ns": 741959, "compute_span_ns": 741959, "calibration_cpu_ns": 2384084, "calibration_gpu_ticks": 2384084, "compute_passes": 1, "command_buffer_ns": 742041.680496, "command_buffers": 2 }, { "compute_ns": 655500, "compute_span_ns": 655500, "calibration_cpu_ns": 2144125, "calibration_gpu_ticks": 2144125, "compute_passes": 1, "command_buffer_ns": 655541.662127, "command_buffers": 2 }, { "compute_ns": 266375, "compute_span_ns": 266375, "calibration_cpu_ns": 821750, "calibration_gpu_ticks": 821750, "compute_passes": 1, "command_buffer_ns": 266499.991994, "command_buffers": 2 }, { "compute_ns": 652292, "compute_span_ns": 652292, "calibration_cpu_ns": 1947792, "calibration_gpu_ticks": 1947792, "compute_passes": 1, "command_buffer_ns": 652333.372273, "command_buffers": 2 } ], "latency": [ { "compute_ns": 11708, "compute_span_ns": 11708, "calibration_cpu_ns": 325041, "calibration_gpu_ticks": 325041, "compute_passes": 1, "command_buffer_ns": 11791.7079479, "command_buffers": 2 }, { "compute_ns": 10291, "compute_span_ns": 10291, "calibration_cpu_ns": 420167, "calibration_gpu_ticks": 420167, "compute_passes": 1, "command_buffer_ns": 10250.0198409, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 394292, "calibration_gpu_ticks": 394292, "compute_passes": 1, "command_buffer_ns": 6583.34465697, "command_buffers": 2 }, { "compute_ns": 6916, "compute_span_ns": 6916, "calibration_cpu_ns": 418625, "calibration_gpu_ticks": 418625, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 6792, "compute_span_ns": 6792, "calibration_cpu_ns": 439750, "calibration_gpu_ticks": 439750, "compute_passes": 1, "command_buffer_ns": 6916.69993103, "command_buffers": 2 }, { "compute_ns": 6875, "compute_span_ns": 6875, "calibration_cpu_ns": 439959, "calibration_gpu_ticks": 439959, "compute_passes": 1, "command_buffer_ns": 6916.64172336, "command_buffers": 2 }, { "compute_ns": 7333, "compute_span_ns": 7333, "calibration_cpu_ns": 831667, "calibration_gpu_ticks": 831667, "compute_passes": 1, "command_buffer_ns": 7416.64553061, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 402875, "calibration_gpu_ticks": 402875, "compute_passes": 1, "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "compute_ns": 7666, "compute_span_ns": 7666, "calibration_cpu_ns": 723958, "calibration_gpu_ticks": 723958, "compute_passes": 1, "command_buffer_ns": 7708.26591179, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 548291.66038, "command_buffers": 2 }, { "command_buffer_ns": 293624.994811, "command_buffers": 2 }, { "command_buffer_ns": 679583.288729, "command_buffers": 2 }, { "command_buffer_ns": 618791.673332, "command_buffers": 2 }, { "command_buffer_ns": 291791.686323, "command_buffers": 2 }, { "command_buffer_ns": 290125.026368, "command_buffers": 2 }, { "command_buffer_ns": 244291.67388, "command_buffers": 2 }, { "command_buffer_ns": 649749.999866, "command_buffers": 2 }, { "command_buffer_ns": 258458.312601, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "command_buffer_ns": 9083.30548555, "command_buffers": 2 }, { "command_buffer_ns": 7750.00080466, "command_buffers": 2 }, { "command_buffer_ns": 17708.3420567, "command_buffers": 2 }, { "command_buffer_ns": 7916.5911302, "command_buffers": 2 }, { "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "command_buffer_ns": 97500.0439212, "command_buffers": 2 }, { "command_buffer_ns": 6874.96503815, "command_buffers": 2 }, { "command_buffer_ns": 69624.9771863, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 8.5670571934375, 4.587890543921875, 10.618488886390626, 9.668619895812501, 4.559245098796875, 4.533203537, 3.817057404375, 10.15234374790625, 4.038411134390625 ], "command_buffer_throughput_us_p50": 4.587890543921875, "command_buffer_throughput_us_p90": 10.245572775603126, "command_buffer_latency_us": [ 6.500049494210001, 9.08330548555, 7.75000080466, 17.708342056699998, 7.9165911302000005, 6.99999509379, 97.5000439212, 6.87496503815, 69.6249771863 ], "command_buffer_latency_us_p50": 7.9165911302000005, "command_buffer_latency_us_p90": 75.19999053328 }, "compute_throughput_us": [ 8.384125, 4.55403125, 4.140625, 3.862640625, 4.272140625, 11.593109375, 10.2421875, 4.162109375, 10.1920625 ], "compute_throughput_us_p50": 4.55403125, "compute_throughput_us_p90": 10.512371875, "command_buffer_throughput_us": [ 8.3854174590625, 4.55468762084375, 4.141276804140626, 3.864583959515625, 4.274088496453125, 11.59440125775, 10.242838470734375, 4.1640623749062495, 10.192708941765625 ], "command_buffer_throughput_us_p50": 4.55468762084375, "command_buffer_throughput_us_p90": 10.5131510281375, "compute_latency_us": [ 11.708, 10.291, 6.5, 6.916, 6.792, 6.875, 7.333, 6.5, 7.666 ], "compute_latency_us_p50": 6.916, "compute_latency_us_p90": 10.5744, "command_buffer_latency_us": [ 11.7917079479, 10.2500198409, 6.58334465697, 6.99999509379, 6.91669993103, 6.91664172336, 7.416645530609999, 6.49999128655, 7.70826591179 ], "command_buffer_latency_us_p50": 6.99999509379, "command_buffer_latency_us_p90": 10.5583574623, "counter_control_throughput_ratio": 0.9787978847025629, "counter_control_latency_ratio": 0.8736967486730947 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 5.84887393573, "throughput_us_p90": 6.3468988190080005, "latency_us_p50": 199.833, "latency_us_p90": 222.725 }, "torch": { "cold_call_ms": 70.993209, "warmup_ms": 200.707, "repetitions": 2832, "throughput_us": [ 9.487964689265537, 10.844338276836158, 9.165533898305085, 9.23268290960452, 9.429393008474577, 9.923066737288135, 9.373234463276836, 9.081832627118644, 9.13382768361582 ], "latency_us": [ 352.333, 373.542, 253.5, 247.417, 255.166, 256.334, 239.042, 232.333, 1444.709 ], "throughput_us_p50": 9.373234463276836, "throughput_us_p90": 10.10732104519774, "latency_us_p50": 255.166, "latency_us_p90": 587.7754000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 493041.6471324861, "command_buffers": 1 }, { "command_buffer_ns": 631041.7084023356, "command_buffers": 1 }, { "command_buffer_ns": 293916.673399508, "command_buffers": 1 }, { "command_buffer_ns": 306333.3570025861, "command_buffers": 1 }, { "command_buffer_ns": 307000.00934302807, "command_buffers": 1 }, { "command_buffer_ns": 539875.0072345138, "command_buffers": 1 }, { "command_buffer_ns": 289250.0488087535, "command_buffers": 1 }, { "command_buffer_ns": 571999.9899156392, "command_buffers": 1 }, { "command_buffer_ns": 335249.9916218221, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 7958.326023072004, "command_buffers": 1 }, { "command_buffer_ns": 103166.61791875958, "command_buffers": 1 }, { "command_buffer_ns": 9250.01222640276, "command_buffers": 1 }, { "command_buffer_ns": 111875.00786036253, "command_buffers": 1 }, { "command_buffer_ns": 9291.688911616802, "command_buffers": 1 }, { "command_buffer_ns": 8374.976459890604, "command_buffers": 1 }, { "command_buffer_ns": 9666.66266322136, "command_buffers": 1 }, { "command_buffer_ns": 7874.972652643919, "command_buffers": 1 }, { "command_buffer_ns": 8374.976459890604, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 7.703775736445095, 9.860026693786494, 4.5924480218673125, 4.786458703165408, 4.796875145984814, 8.435546988039277, 4.519532012636773, 8.937499842431862, 5.238281119090971 ], "command_buffer_throughput_us_p50": 5.238281119090971, "command_buffer_throughput_us_p90": 9.122005212702788, "command_buffer_latency_us": [ 7.958326023072004, 103.16661791875958, 9.25001222640276, 111.87500786036253, 9.291688911616802, 8.374976459890604, 9.66666266322136, 7.874972652643919, 8.374976459890604 ], "command_buffer_latency_us_p50": 9.25001222640276, "command_buffer_latency_us_p90": 104.90829590708017 }, "throughput": [ { "compute_ns": 330416.0, "compute_span_ns": 330416.0, "command_buffer_ns": 330374.9836049974, "calibration_cpu_ns": 925250.0, "calibration_gpu_ticks": 925250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 290417.0, "compute_span_ns": 290417.0, "command_buffer_ns": 290458.38164165616, "calibration_cpu_ns": 930792.0, "calibration_gpu_ticks": 930792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289458.0, "compute_span_ns": 289458.0, "command_buffer_ns": 289458.3158195019, "calibration_cpu_ns": 994750.0, "calibration_gpu_ticks": 994750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 505792.0, "compute_span_ns": 505792.0, "command_buffer_ns": 505833.3626948297, "calibration_cpu_ns": 1602750.0, "calibration_gpu_ticks": 1602750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 458333.0, "compute_span_ns": 458333.0, "command_buffer_ns": 458333.3502523601, "calibration_cpu_ns": 2079166.0, "calibration_gpu_ticks": 2079166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289167.0, "compute_span_ns": 289167.0, "command_buffer_ns": 289166.6372306645, "calibration_cpu_ns": 1005291.0, "calibration_gpu_ticks": 1005291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 507750.0, "compute_span_ns": 507750.0, "command_buffer_ns": 507750.02455338836, "calibration_cpu_ns": 2026458.0, "calibration_gpu_ticks": 2026458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 323417.0, "compute_span_ns": 323417.0, "command_buffer_ns": 323458.3418816328, "calibration_cpu_ns": 982583.0, "calibration_gpu_ticks": 982583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 526208.0, "compute_span_ns": 526208.0, "command_buffer_ns": 526166.6956357658, "calibration_cpu_ns": 2019208.0, "calibration_gpu_ticks": 2019208.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 8334.0, "compute_span_ns": 8334.0, "command_buffer_ns": 8374.976459890604, "calibration_cpu_ns": 466917.0, "calibration_gpu_ticks": 466917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8500.0, "compute_span_ns": 8500.0, "command_buffer_ns": 8500.006515532732, "calibration_cpu_ns": 410791.0, "calibration_gpu_ticks": 410791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8124.9745562672615, "calibration_cpu_ns": 435709.0, "calibration_gpu_ticks": 435709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8334.0, "compute_span_ns": 8334.0, "command_buffer_ns": 8375.034667551517, "calibration_cpu_ns": 425125.0, "calibration_gpu_ticks": 425125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7875.0, "compute_span_ns": 7875.0, "command_buffer_ns": 7875.0308603048325, "calibration_cpu_ns": 446667.0, "calibration_gpu_ticks": 446667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8292.0, "compute_span_ns": 8292.0, "command_buffer_ns": 8291.681297123432, "calibration_cpu_ns": 469333.0, "calibration_gpu_ticks": 469333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10125.0, "compute_span_ns": 10125.0, "command_buffer_ns": 10124.989785254002, "calibration_cpu_ns": 783166.0, "calibration_gpu_ticks": 783166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8250.0, "compute_span_ns": 8250.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 435042.0, "calibration_gpu_ticks": 435042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9666.0, "compute_span_ns": 9666.0, "command_buffer_ns": 9624.985978007317, "calibration_cpu_ns": 1883167.0, "calibration_gpu_ticks": 1883167.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 5.16275, 4.537765625, 4.52278125, 7.903, 7.161453125, 4.518234375, 7.93359375, 5.053390625, 8.222 ], "compute_throughput_us_p50": 5.16275, "compute_throughput_us_p90": 7.991275, "command_buffer_throughput_us": [ 5.162109118828084, 4.5384122131508775, 4.522786184679717, 7.903646292106714, 7.161458597693127, 4.518228706729133, 7.933594133646693, 5.054036591900513, 8.22135461930884 ], "command_buffer_throughput_us_p50": 5.162109118828084, "command_buffer_throughput_us_p90": 7.991146230779123, "compute_latency_us": [ 8.334, 8.5, 8.125, 8.334, 7.875, 8.292, 10.125, 8.25, 9.666 ], "compute_latency_us_p50": 8.334, "compute_latency_us_p90": 9.7578, "command_buffer_latency_us": [ 8.374976459890604, 8.500006515532732, 8.124974556267262, 8.375034667551517, 7.8750308603048325, 8.291681297123432, 10.124989785254002, 8.25000461190939, 9.624985978007317 ], "command_buffer_latency_us_p50": 8.374976459890604, "command_buffer_latency_us_p90": 9.724986739456654, "counter_control_throughput_ratio": 0.9848312192416996, "counter_control_latency_ratio": 0.9900542809683002 }, "download_ms": 0.518791, "allocation_upload_ms": 1.12425, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.6239974001125396, "device_compute_slowdown": 0.8820940874533921, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.69325, "warmup_ms": 200.096042, "repetitions": 2547, "throughput_us": [ 11.132050647820966, 10.131968983117392, 9.574728307813112, 9.681782489202984, 9.808745583038869, 9.873789556340792, 9.772739301138593, 9.834085983510011, 10.145956026698077 ], "latency_us": [ 362.458, 220.125, 201.792, 212.167, 221.167, 227.458, 219.958, 212.375, 224.375 ], "throughput_us_p50": 9.834085983510011, "throughput_us_p90": 10.343174950922656, "latency_us_p50": 220.125, "latency_us_p90": 254.45800000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 536833.3659134805, "command_buffers": 1 }, { "command_buffer_ns": 651124.9812319875, "command_buffers": 1 }, { "command_buffer_ns": 332875.0026412308, "command_buffers": 1 }, { "command_buffer_ns": 345041.6843406856, "command_buffers": 1 }, { "command_buffer_ns": 289999.9963119626, "command_buffers": 1 }, { "command_buffer_ns": 289916.6429415345, "command_buffers": 1 }, { "command_buffer_ns": 533291.6625775397, "command_buffers": 1 }, { "command_buffer_ns": 602125.0155754387, "command_buffers": 1 }, { "command_buffer_ns": 366375.02489611506, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 79249.96316432953, "command_buffers": 1 }, { "command_buffer_ns": 7833.295967429876, "command_buffers": 1 }, { "command_buffer_ns": 81541.65698215365, "command_buffers": 1 }, { "command_buffer_ns": 8375.034667551517, "command_buffers": 1 }, { "command_buffer_ns": 73916.68623313308, "command_buffers": 1 }, { "command_buffer_ns": 11625.001206994057, "command_buffers": 1 }, { "command_buffer_ns": 8250.00461190939, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 8250.00461190939, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 8.388021342398133, 10.173827831749804, 5.201171916269232, 5.391276317823213, 4.531249942374416, 4.529947545961477, 8.332682227774058, 9.40820336836623, 5.724609764001798 ], "command_buffer_throughput_us_p50": 5.724609764001798, "command_buffer_throughput_us_p90": 9.561328261042945, "command_buffer_latency_us": [ 79.24996316432953, 7.833295967429876, 81.54165698215365, 8.375034667551517, 73.91668623313308, 11.625001206994057, 8.25000461190939, 8.124974556267262, 8.25000461190939 ], "command_buffer_latency_us_p50": 8.375034667551517, "command_buffer_latency_us_p90": 79.70830192789435 }, "throughput": [ { "compute_ns": 337708.0, "compute_span_ns": 337708.0, "command_buffer_ns": 337666.65728762746, "calibration_cpu_ns": 937792.0, "calibration_gpu_ticks": 937792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 291000.0, "compute_span_ns": 291000.0, "command_buffer_ns": 291000.003926456, "calibration_cpu_ns": 910500.0, "calibration_gpu_ticks": 910500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 306833.0, "compute_span_ns": 306833.0, "command_buffer_ns": 306833.3608098328, "calibration_cpu_ns": 940083.0, "calibration_gpu_ticks": 940083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 507250.0, "compute_span_ns": 507250.0, "command_buffer_ns": 507249.96253848076, "calibration_cpu_ns": 1882750.0, "calibration_gpu_ticks": 1882750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 678375.0, "compute_span_ns": 678375.0, "command_buffer_ns": 678374.9558962882, "calibration_cpu_ns": 2006291.0, "calibration_gpu_ticks": 2006291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 303208.0, "compute_span_ns": 303208.0, "command_buffer_ns": 303208.3041034639, "calibration_cpu_ns": 1038250.0, "calibration_gpu_ticks": 1038250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 344459.0, "compute_span_ns": 344459.0, "command_buffer_ns": 344500.0038482249, "calibration_cpu_ns": 959958.0, "calibration_gpu_ticks": 959958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 306750.0, "compute_span_ns": 306750.0, "command_buffer_ns": 306750.0074394047, "calibration_cpu_ns": 908375.0, "calibration_gpu_ticks": 908375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 291292.0, "compute_span_ns": 291292.0, "command_buffer_ns": 291291.68251529336, "calibration_cpu_ns": 934458.0, "calibration_gpu_ticks": 934458.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 9250.0, "compute_span_ns": 9250.0, "command_buffer_ns": 9250.01222640276, "calibration_cpu_ns": 751291.0, "calibration_gpu_ticks": 751291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8625.0, "compute_span_ns": 8625.0, "command_buffer_ns": 8624.978363513947, "calibration_cpu_ns": 395042.0, "calibration_gpu_ticks": 395042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9417.0, "compute_span_ns": 9417.0, "command_buffer_ns": 9416.660759598017, "calibration_cpu_ns": 1251792.0, "calibration_gpu_ticks": 1251792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8750.0, "compute_span_ns": 8750.0, "command_buffer_ns": 8750.008419156075, "calibration_cpu_ns": 505000.0, "calibration_gpu_ticks": 505000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9417.0, "compute_span_ns": 9417.0, "command_buffer_ns": 9458.33744481206, "calibration_cpu_ns": 804917.0, "calibration_gpu_ticks": 804917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7958.0, "compute_span_ns": 7958.0, "command_buffer_ns": 7916.649337857962, "calibration_cpu_ns": 391917.0, "calibration_gpu_ticks": 391917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81708.0, "compute_span_ns": 81708.0, "command_buffer_ns": 81708.36372300982, "calibration_cpu_ns": 1231625.0, "calibration_gpu_ticks": 1231625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9083.0, "compute_span_ns": 9083.0, "command_buffer_ns": 9041.628800332546, "calibration_cpu_ns": 849375.0, "calibration_gpu_ticks": 849375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8333.0, "compute_span_ns": 8333.0, "command_buffer_ns": 8291.681297123432, "calibration_cpu_ns": 402166.0, "calibration_gpu_ticks": 402166.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 5.2766875, 4.546875, 4.794265625, 7.92578125, 10.599609375, 4.737625, 5.382171875, 4.79296875, 4.5514375 ], "compute_throughput_us_p50": 4.794265625, "compute_throughput_us_p90": 8.460546875, "command_buffer_throughput_us": [ 5.276041520119179, 4.546875061350875, 4.794271262653638, 7.925780664663762, 10.599608685879502, 4.737629751616623, 5.382812560128514, 4.792968866240699, 4.551432539301459 ], "command_buffer_throughput_us_p50": 4.794271262653638, "command_buffer_throughput_us_p90": 8.46054626890691, "compute_latency_us": [ 9.25, 8.625, 9.417, 8.75, 9.417, 7.958, 81.708, 9.083, 8.333 ], "compute_latency_us_p50": 9.083, "compute_latency_us_p90": 23.875200000000014, "command_buffer_latency_us": [ 9.25001222640276, 8.624978363513947, 9.416660759598017, 8.750008419156075, 9.45833744481206, 7.916649337857962, 81.70836372300982, 9.041628800332546, 8.291681297123432 ], "command_buffer_latency_us_p50": 9.041628800332546, "command_buffer_latency_us_p90": 23.908342700451627, "counter_control_throughput_ratio": 0.7950642448892049, "counter_control_latency_ratio": 1.0050517165958768 }, "download_ms": 0.318458, "allocation_upload_ms": 3.046125, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-l514mu8x/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "e81310ef6c7b3435af776cabf332187d02f0ddc801b9234c32af8a56116f248b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.158208, "capture_ms": 0.065625, "compile_ms": 64.572083, "allocation_upload_ms": 1.945833, "cold_call_ms": 2.045084, "warmup_ms": 200.131125, "download_ms": 0.366959, "repetitions": 3990, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.32488496241, 6.29597969925, 6.68083759398, 6.1652358396, 7.10079373434, 6.36380526316, 6.30533634085, 6.31369047619, 6.33959899749 ], "latency_us": [ 252.25, 244.042, 224.667, 214.375, 196.292, 229.792, 287.708, 230.708, 233.167 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 48, "normalized_kernel_cost": 48, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 265875, "compute_span_ns": 265875, "calibration_cpu_ns": 832542, "calibration_gpu_ticks": 832542, "compute_passes": 1, "command_buffer_ns": 265916.693024, "command_buffers": 2 }, { "compute_ns": 292667, "compute_span_ns": 292667, "calibration_cpu_ns": 776417, "calibration_gpu_ticks": 776417, "compute_passes": 1, "command_buffer_ns": 292791.635729, "command_buffers": 2 }, { "compute_ns": 600167, "compute_span_ns": 600167, "calibration_cpu_ns": 1833208, "calibration_gpu_ticks": 1833208, "compute_passes": 1, "command_buffer_ns": 600333.325565, "command_buffers": 2 }, { "compute_ns": 281625, "compute_span_ns": 281625, "calibration_cpu_ns": 759041, "calibration_gpu_ticks": 759041, "compute_passes": 1, "command_buffer_ns": 281708.373223, "command_buffers": 2 }, { "compute_ns": 265083, "compute_span_ns": 265083, "calibration_cpu_ns": 737875, "calibration_gpu_ticks": 737875, "compute_passes": 1, "command_buffer_ns": 265125.010628, "command_buffers": 2 }, { "compute_ns": 265667, "compute_span_ns": 265667, "calibration_cpu_ns": 731791, "calibration_gpu_ticks": 731791, "compute_passes": 1, "command_buffer_ns": 265750.044491, "command_buffers": 2 }, { "compute_ns": 292833, "compute_span_ns": 292833, "calibration_cpu_ns": 749584, "calibration_gpu_ticks": 749584, "compute_passes": 1, "command_buffer_ns": 292874.9891, "command_buffers": 2 }, { "compute_ns": 567250, "compute_span_ns": 567250, "calibration_cpu_ns": 2144000, "calibration_gpu_ticks": 2144000, "compute_passes": 1, "command_buffer_ns": 567333.365325, "command_buffers": 2 }, { "compute_ns": 292125, "compute_span_ns": 292125, "calibration_cpu_ns": 742958, "calibration_gpu_ticks": 742958, "compute_passes": 1, "command_buffer_ns": 292166.660074, "command_buffers": 2 } ], "latency": [ { "compute_ns": 11042, "compute_span_ns": 11042, "calibration_cpu_ns": 907417, "calibration_gpu_ticks": 907417, "compute_passes": 1, "command_buffer_ns": 11083.3207145, "command_buffers": 2 }, { "compute_ns": 7750, "compute_span_ns": 7750, "calibration_cpu_ns": 821750, "calibration_gpu_ticks": 821750, "compute_passes": 1, "command_buffer_ns": 7791.61928222, "command_buffers": 2 }, { "compute_ns": 7167, "compute_span_ns": 7167, "calibration_cpu_ns": 484167, "calibration_gpu_ticks": 484167, "compute_passes": 1, "command_buffer_ns": 7249.99699742, "command_buffers": 2 }, { "compute_ns": 92250, "compute_span_ns": 92250, "calibration_cpu_ns": 987333, "calibration_gpu_ticks": 987333, "compute_passes": 1, "command_buffer_ns": 92375.0340007, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 405125, "calibration_gpu_ticks": 405125, "compute_passes": 1, "command_buffer_ns": 7083.34846422, "command_buffers": 2 }, { "compute_ns": 124542, "compute_span_ns": 124542, "calibration_cpu_ns": 1544583, "calibration_gpu_ticks": 1544583, "compute_passes": 1, "command_buffer_ns": 124666.723423, "command_buffers": 2 }, { "compute_ns": 77000, "compute_span_ns": 77000, "calibration_cpu_ns": 1265250, "calibration_gpu_ticks": 1265250, "compute_passes": 1, "command_buffer_ns": 77000.0042394, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 794084, "calibration_gpu_ticks": 794084, "compute_passes": 1, "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "compute_ns": 42375, "compute_span_ns": 42375, "calibration_cpu_ns": 1200625, "calibration_gpu_ticks": 1200625, "compute_passes": 1, "command_buffer_ns": 42499.97437, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 701958.255377, "command_buffers": 2 }, { "command_buffer_ns": 774083.309807, "command_buffers": 2 }, { "command_buffer_ns": 280416.687019, "command_buffers": 2 }, { "command_buffer_ns": 281541.666482, "command_buffers": 2 }, { "command_buffer_ns": 611249.997746, "command_buffers": 2 }, { "command_buffer_ns": 711500.062607, "command_buffers": 2 }, { "command_buffer_ns": 288458.308205, "command_buffers": 2 }, { "command_buffer_ns": 267166.702542, "command_buffers": 2 }, { "command_buffer_ns": 691374.938469, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "command_buffer_ns": 6624.96313453, "command_buffers": 2 }, { "command_buffer_ns": 7249.99699742, "command_buffers": 2 }, { "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "command_buffer_ns": 6666.6980274, "command_buffers": 2 }, { "command_buffer_ns": 7166.64362699, "command_buffers": 2 }, { "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "command_buffer_ns": 6458.31460133, "command_buffers": 2 }, { "command_buffer_ns": 7083.34846422, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 10.968097740265625, 12.095051715734375, 4.381510734671875, 4.39908853878125, 9.55078121478125, 11.117188478234375, 4.507161065703125, 4.17447972721875, 10.802733413578126 ], "command_buffer_throughput_us_p50": 9.55078121478125, "command_buffer_throughput_us_p90": 11.312761125734376, "command_buffer_latency_us": [ 7.1249669417699995, 6.624963134530001, 7.24999699742, 7.708324119449999, 6.6666980274, 7.16664362699, 6.54166797176, 6.458314601330001, 7.08334846422 ], "command_buffer_latency_us_p50": 7.08334846422, "command_buffer_latency_us_p90": 7.341662421826 }, "compute_throughput_us": [ 4.154296875, 4.572921875, 9.377609375, 4.400390625, 4.141921875, 4.151046875, 4.575515625, 8.86328125, 4.564453125 ], "compute_throughput_us_p50": 4.564453125, "compute_throughput_us_p90": 8.966146875, "command_buffer_throughput_us": [ 4.1549483285, 4.574869308265624, 9.380208211953125, 4.4016933316093745, 4.142578291062501, 4.152344445171875, 4.5761717046875, 8.864583833203126, 4.56510406365625 ], "command_buffer_throughput_us_p50": 4.56510406365625, "command_buffer_throughput_us_p90": 8.967708708953126, "compute_latency_us": [ 11.042, 7.75, 7.167, 92.25, 6.958, 124.542, 77.0, 7.25, 42.375 ], "compute_latency_us_p50": 11.042, "compute_latency_us_p90": 98.70840000000001, "command_buffer_latency_us": [ 11.0833207145, 7.79161928222, 7.24999699742, 92.37503400070001, 7.08334846422, 124.66672342300001, 77.0000042394, 7.54167558625, 42.499974370000004 ], "command_buffer_latency_us_p50": 11.0833207145, "command_buffer_latency_us_p90": 98.83337188516002, "counter_control_throughput_ratio": 0.4337423502751007, "counter_control_latency_ratio": 1.5555610019078427 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 6.32488496241, "throughput_us_p90": 6.764828822051999, "latency_us_p50": 230.708, "latency_us_p90": 259.3416 }, "slowdown": 0.6431594123760653, "device_compute_slowdown": 0.9520651298915044, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-5rlsc4dr/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "a8c235e7c8be57d97298e1f9f70525f6c2c7f4ae4cee9647b15503a46aab5ade", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.7505, "capture_ms": 0.079917, "compile_ms": 57.31525, "allocation_upload_ms": 2.289708, "cold_call_ms": 6.135417, "warmup_ms": 200.23625, "download_ms": 0.92775, "repetitions": 1577, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 23.3050095117, 21.5134483196, 21.5564629042, 21.6622276474, 22.4908319594, 23.070439442, 21.9010779962, 21.7224426126, 22.1561769182 ], "latency_us": [ 300.417, 252.5, 263.833, 256.291, 245.458, 445.916, 2048.084, 1005.834, 541.416 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 60, "normalized_kernel_cost": 240, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 985125, "compute_span_ns": 985125, "calibration_cpu_ns": 1448458, "calibration_gpu_ticks": 1448458, "compute_passes": 1, "command_buffer_ns": 985166.698229, "command_buffers": 2 }, { "compute_ns": 1313333, "compute_span_ns": 1313333, "calibration_cpu_ns": 2564875, "calibration_gpu_ticks": 2564875, "compute_passes": 1, "command_buffer_ns": 1313333.34139, "command_buffers": 2 }, { "compute_ns": 960833, "compute_span_ns": 960833, "calibration_cpu_ns": 1617708, "calibration_gpu_ticks": 1617708, "compute_passes": 1, "command_buffer_ns": 960875.011515, "command_buffers": 2 }, { "compute_ns": 1249458, "compute_span_ns": 1249458, "calibration_cpu_ns": 2470208, "calibration_gpu_ticks": 2470208, "compute_passes": 1, "command_buffer_ns": 1249541.64494, "command_buffers": 2 }, { "compute_ns": 957209, "compute_span_ns": 957209, "calibration_cpu_ns": 1464958, "calibration_gpu_ticks": 1464958, "compute_passes": 1, "command_buffer_ns": 957374.984864, "command_buffers": 2 }, { "compute_ns": 1462833, "compute_span_ns": 1462833, "calibration_cpu_ns": 2729875, "calibration_gpu_ticks": 2729875, "compute_passes": 1, "command_buffer_ns": 1462833.3156, "command_buffers": 2 }, { "compute_ns": 962041, "compute_span_ns": 962041, "calibration_cpu_ns": 1443042, "calibration_gpu_ticks": 1443042, "compute_passes": 1, "command_buffer_ns": 962083.344348, "command_buffers": 2 }, { "compute_ns": 1239833, "compute_span_ns": 1239833, "calibration_cpu_ns": 2553916, "calibration_gpu_ticks": 2553916, "compute_passes": 1, "command_buffer_ns": 1239791.68711, "command_buffers": 2 }, { "compute_ns": 970875, "compute_span_ns": 970875, "calibration_cpu_ns": 1454792, "calibration_gpu_ticks": 1454792, "compute_passes": 1, "command_buffer_ns": 970916.64793, "command_buffers": 2 } ], "latency": [ { "compute_ns": 20709, "compute_span_ns": 20709, "calibration_cpu_ns": 1112875, "calibration_gpu_ticks": 1112875, "compute_passes": 1, "command_buffer_ns": 20833.2785405, "command_buffers": 2 }, { "compute_ns": 19125, "compute_span_ns": 19125, "calibration_cpu_ns": 429209, "calibration_gpu_ticks": 429209, "compute_passes": 1, "command_buffer_ns": 19208.2952708, "command_buffers": 2 }, { "compute_ns": 22917, "compute_span_ns": 22917, "calibration_cpu_ns": 768292, "calibration_gpu_ticks": 768292, "compute_passes": 1, "command_buffer_ns": 23041.6771956, "command_buffers": 2 }, { "compute_ns": 16708, "compute_span_ns": 16708, "calibration_cpu_ns": 529583, "calibration_gpu_ticks": 529583, "compute_passes": 1, "command_buffer_ns": 16749.9529198, "command_buffers": 2 }, { "compute_ns": 21375, "compute_span_ns": 21375, "calibration_cpu_ns": 883000, "calibration_gpu_ticks": 883000, "compute_passes": 1, "command_buffer_ns": 21416.6357182, "command_buffers": 2 }, { "compute_ns": 17084, "compute_span_ns": 17084, "calibration_cpu_ns": 470125, "calibration_gpu_ticks": 470125, "compute_passes": 1, "command_buffer_ns": 17208.3382495, "command_buffers": 2 }, { "compute_ns": 20875, "compute_span_ns": 20875, "calibration_cpu_ns": 819959, "calibration_gpu_ticks": 819959, "compute_passes": 1, "command_buffer_ns": 21000.043489, "command_buffers": 2 }, { "compute_ns": 17125, "compute_span_ns": 17125, "calibration_cpu_ns": 452916, "calibration_gpu_ticks": 452916, "compute_passes": 1, "command_buffer_ns": 17250.0149347, "command_buffers": 2 }, { "compute_ns": 21334, "compute_span_ns": 21334, "calibration_cpu_ns": 840625, "calibration_gpu_ticks": 840625, "compute_passes": 1, "command_buffer_ns": 21416.6939259, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1395083.32359, "command_buffers": 2 }, { "command_buffer_ns": 953541.661147, "command_buffers": 2 }, { "command_buffer_ns": 1486458.34997, "command_buffers": 2 }, { "command_buffer_ns": 962375.022937, "command_buffers": 2 }, { "command_buffer_ns": 1343333.27878, "command_buffers": 2 }, { "command_buffer_ns": 962708.261795, "command_buffers": 2 }, { "command_buffer_ns": 1448666.61867, "command_buffers": 2 }, { "command_buffer_ns": 967291.649431, "command_buffers": 2 }, { "command_buffer_ns": 1497208.31541, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 50083.2684338, "command_buffers": 2 }, { "command_buffer_ns": 19041.5885299, "command_buffers": 2 }, { "command_buffer_ns": 63208.3974779, "command_buffers": 2 }, { "command_buffer_ns": 19625.0039153, "command_buffers": 2 }, { "command_buffer_ns": 84708.2701512, "command_buffers": 2 }, { "command_buffer_ns": 19333.3253264, "command_buffers": 2 }, { "command_buffer_ns": 42916.6830145, "command_buffers": 2 }, { "command_buffer_ns": 19416.6786969, "command_buffers": 2 }, { "command_buffer_ns": 25875.0515059, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 21.79817693109375, 14.899088455421875, 23.225911718281253, 15.037109733390626, 20.9895824809375, 15.042316590546877, 22.63541591671875, 15.113932022359375, 23.39387992828125 ], "command_buffer_throughput_us_p50": 20.9895824809375, "command_buffer_throughput_us_p90": 23.259505360281253, "command_buffer_latency_us": [ 50.0832684338, 19.0415885299, 63.2083974779, 19.6250039153, 84.70827015120001, 19.3333253264, 42.9166830145, 19.4166786969, 25.8750515059 ], "command_buffer_latency_us_p50": 25.8750515059, "command_buffer_latency_us_p90": 67.50837201256 }, "compute_throughput_us": [ 15.392578125, 20.520828125, 15.013015625, 19.52278125, 14.956390625, 22.856765625, 15.031890625, 19.372390625, 15.169921875 ], "compute_throughput_us_p50": 15.392578125, "compute_throughput_us_p90": 20.988015625000003, "command_buffer_throughput_us": [ 15.393229659828124, 20.52083345921875, 15.013672054921875, 19.5240882021875, 14.9589841385, 22.85677055625, 15.0325522554375, 19.37174511109375, 15.17057262390625 ], "command_buffer_throughput_us_p50": 15.393229659828124, "command_buffer_throughput_us_p90": 20.988020878625, "compute_latency_us": [ 20.709, 19.125, 22.917, 16.708, 21.375, 17.084, 20.875, 17.125, 21.334 ], "compute_latency_us_p50": 20.709, "compute_latency_us_p90": 21.6834, "command_buffer_latency_us": [ 20.8332785405, 19.2082952708, 23.041677195600002, 16.7499529198, 21.4166357182, 17.2083382495, 21.000043489, 17.2500149347, 21.4166939259 ], "command_buffer_latency_us_p50": 20.8332785405, "command_buffer_latency_us_p90": 21.74169057984, "counter_control_throughput_ratio": 0.7126861218933525, "counter_control_latency_ratio": 0.8276966683918132 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 21.9010779962, "throughput_us_p90": 23.117353455940002, "latency_us_p50": 300.417, "latency_us_p90": 1214.284 }, "torch": { "cold_call_ms": 6.579667, "warmup_ms": 200.59575, "repetitions": 1109, "throughput_us": [ 25.963931469792605, 26.937743913435526, 26.67684851217313, 26.27821641118124, 27.00364472497746, 26.58427321911632, 28.52678809738503, 26.493537421100093, 27.777427412082957 ], "latency_us": [ 365.209, 284.125, 286.95799999999997, 263.958, 260.541, 261.084, 279.792, 3117.333, 474.375 ], "throughput_us_p50": 26.67684851217313, "throughput_us_p90": 27.92729954914337, "latency_us_p50": 284.125, "latency_us_p90": 1002.9666000000005, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1081124.9958351254, "command_buffers": 1 }, { "command_buffer_ns": 1147250.0045783818, "command_buffers": 1 }, { "command_buffer_ns": 1034749.9628551304, "command_buffers": 1 }, { "command_buffer_ns": 1459416.642319411, "command_buffers": 1 }, { "command_buffer_ns": 1061999.995727092, "command_buffers": 1 }, { "command_buffer_ns": 1320499.9850131571, "command_buffers": 1 }, { "command_buffer_ns": 1035333.3200328052, "command_buffers": 1 }, { "command_buffer_ns": 1321083.342190832, "command_buffers": 1 }, { "command_buffer_ns": 1038916.6418462992, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 18791.64483398199, "command_buffers": 1 }, { "command_buffer_ns": 21291.663870215416, "command_buffers": 1 }, { "command_buffer_ns": 19125.00010803342, "command_buffers": 1 }, { "command_buffer_ns": 25124.987587332726, "command_buffers": 1 }, { "command_buffer_ns": 26124.995201826096, "command_buffers": 1 }, { "command_buffer_ns": 85624.9826028943, "command_buffers": 1 }, { "command_buffer_ns": 21249.987185001373, "command_buffers": 1 }, { "command_buffer_ns": 28499.984182417393, "command_buffers": 1 }, { "command_buffer_ns": 25500.019546598196, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.892578059923835, 17.925781321537215, 16.167968169611413, 22.803385036240797, 16.593749933235813, 20.63281226583058, 16.17708312551258, 20.64192722173175, 16.233072528848425 ], "command_buffer_throughput_us_p50": 16.892578059923835, "command_buffer_throughput_us_p90": 21.07421878463356, "command_buffer_latency_us": [ 18.79164483398199, 21.291663870215416, 19.12500010803342, 25.124987587332726, 26.124995201826096, 85.6249826028943, 21.249987185001373, 28.499984182417393, 25.500019546598196 ], "command_buffer_latency_us_p50": 25.124987587332726, "command_buffer_latency_us_p90": 39.92498386651279 }, "throughput": [ { "compute_ns": 1236541.0, "compute_span_ns": 1236541.0, "command_buffer_ns": 1236499.985679984, "calibration_cpu_ns": 2723458.0, "calibration_gpu_ticks": 2723458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1035541.0, "compute_span_ns": 1035541.0, "command_buffer_ns": 1035500.0267736614, "calibration_cpu_ns": 1847625.0, "calibration_gpu_ticks": 1847625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1185375.0, "compute_span_ns": 1185375.0, "command_buffer_ns": 1185374.9747388065, "calibration_cpu_ns": 2791250.0, "calibration_gpu_ticks": 2791250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1032792.0, "compute_span_ns": 1032792.0, "command_buffer_ns": 1032833.3592042327, "calibration_cpu_ns": 1761334.0, "calibration_gpu_ticks": 1761334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1355084.0, "compute_span_ns": 1355084.0, "command_buffer_ns": 1355125.0449381769, "calibration_cpu_ns": 2642083.0, "calibration_gpu_ticks": 2642083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1031959.0, "compute_span_ns": 1031959.0, "command_buffer_ns": 1032000.0001229346, "calibration_cpu_ns": 1774458.0, "calibration_gpu_ticks": 1774458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1327542.0, "compute_span_ns": 1327542.0, "command_buffer_ns": 1327541.7149998248, "calibration_cpu_ns": 2642333.0, "calibration_gpu_ticks": 2642333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1078500.0, "compute_span_ns": 1078500.0, "command_buffer_ns": 1078500.0049509108, "calibration_cpu_ns": 1685500.0, "calibration_gpu_ticks": 1685500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1300833.0, "compute_span_ns": 1300833.0, "command_buffer_ns": 1300791.6859351099, "calibration_cpu_ns": 2741459.0, "calibration_gpu_ticks": 2741459.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 18666.0, "compute_span_ns": 18666.0, "command_buffer_ns": 18625.054508447647, "calibration_cpu_ns": 446417.0, "calibration_gpu_ticks": 446417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 95583.0, "compute_span_ns": 95583.0, "command_buffer_ns": 95541.70537739992, "calibration_cpu_ns": 1204834.0, "calibration_gpu_ticks": 1204834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20958.0, "compute_span_ns": 20958.0, "command_buffer_ns": 20958.308596163988, "calibration_cpu_ns": 500625.0, "calibration_gpu_ticks": 500625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20875.0, "compute_span_ns": 20875.0, "command_buffer_ns": 20875.013433396816, "calibration_cpu_ns": 415042.0, "calibration_gpu_ticks": 415042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18708.0, "compute_span_ns": 18708.0, "command_buffer_ns": 18666.672986000776, "calibration_cpu_ns": 444584.0, "calibration_gpu_ticks": 444584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21584.0, "compute_span_ns": 21584.0, "command_buffer_ns": 21625.019144266844, "calibration_cpu_ns": 566042.0, "calibration_gpu_ticks": 566042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18667.0, "compute_span_ns": 18667.0, "command_buffer_ns": 18666.672986000776, "calibration_cpu_ns": 439417.0, "calibration_gpu_ticks": 439417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21167.0, "compute_span_ns": 21167.0, "command_buffer_ns": 21166.633814573288, "calibration_cpu_ns": 527625.0, "calibration_gpu_ticks": 527625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18666.0, "compute_span_ns": 18666.0, "command_buffer_ns": 18624.996300786734, "calibration_cpu_ns": 446833.0, "calibration_gpu_ticks": 446833.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 19.320953125, 16.180328125, 18.521484375, 16.137375, 21.1731875, 16.124359375, 20.74284375, 16.8515625, 20.325515625 ], "compute_throughput_us_p50": 18.521484375, "compute_throughput_us_p90": 20.828912499999998, "command_buffer_throughput_us": [ 19.32031227624975, 16.17968791833846, 18.52148398029385, 16.138021237566136, 21.173828827159014, 16.125000001920853, 20.742839296872262, 16.85156257735798, 20.32487009273609 ], "command_buffer_throughput_us_p50": 18.52148398029385, "command_buffer_throughput_us_p90": 20.829037202929612, "compute_latency_us": [ 18.666, 95.583, 20.958, 20.875, 18.708, 21.584, 18.667, 21.167, 18.666 ], "compute_latency_us_p50": 20.875, "compute_latency_us_p90": 36.38380000000001, "command_buffer_latency_us": [ 18.625054508447647, 95.54170537739992, 20.958308596163988, 20.875013433396816, 18.666672986000776, 21.625019144266844, 18.666672986000776, 21.166633814573288, 18.624996300786734 ], "command_buffer_latency_us_p50": 20.875013433396816, "command_buffer_latency_us_p90": 36.40835639089347, "counter_control_throughput_ratio": 1.1437160276965364, "counter_control_latency_ratio": 0.8308467162754493 }, "download_ms": 0.352084, "allocation_upload_ms": 1.647417, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.8209769600860515, "device_compute_slowdown": 0.8310661183169883, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.368875, "warmup_ms": 200.097834, "repetitions": 1203, "throughput_us": [ 26.331463009143807, 28.022789692435577, 27.031275976724853, 27.417463840399005, 27.39314962593516, 27.971356608478796, 27.6880365752286, 28.035224438902745, 27.20109393183707 ], "latency_us": [ 396.292, 285.54200000000003, 278.292, 292.16700000000003, 288.375, 268.25, 277.91700000000003, 260.375, 252.583 ], "throughput_us_p50": 27.417463840399005, "throughput_us_p90": 28.02527664172901, "latency_us_p50": 278.292, "latency_us_p90": 312.992, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1362458.33709836, "command_buffers": 1 }, { "command_buffer_ns": 1035541.6452512145, "command_buffers": 1 }, { "command_buffer_ns": 1302124.9906159937, "command_buffers": 1 }, { "command_buffer_ns": 1043249.9693706632, "command_buffers": 1 }, { "command_buffer_ns": 1383124.9671056867, "command_buffers": 1 }, { "command_buffer_ns": 1042083.3132229745, "command_buffers": 1 }, { "command_buffer_ns": 1316500.0127628446, "command_buffers": 1 }, { "command_buffer_ns": 1032750.0058338046, "command_buffers": 1 }, { "command_buffer_ns": 1311249.9727867544, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 52999.99611452222, "command_buffers": 1 }, { "command_buffer_ns": 21083.338651806116, "command_buffers": 1 }, { "command_buffer_ns": 155916.67033731937, "command_buffers": 1 }, { "command_buffer_ns": 20791.66006296873, "command_buffers": 1 }, { "command_buffer_ns": 99375.02909451723, "command_buffers": 1 }, { "command_buffer_ns": 20875.013433396816, "command_buffers": 1 }, { "command_buffer_ns": 20749.98337775469, "command_buffers": 1 }, { "command_buffer_ns": 18791.64483398199, "command_buffers": 1 }, { "command_buffer_ns": 21125.01533702016, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 21.288411517161876, 16.180338207050227, 20.345702978374902, 16.300780771416612, 21.611327611026354, 16.282551769108977, 20.570312699419446, 16.136718841153197, 20.488280824793037 ], "command_buffer_throughput_us_p50": 20.345702978374902, "command_buffer_throughput_us_p90": 21.35299473593477, "command_buffer_latency_us": [ 52.99999611452222, 21.083338651806116, 155.91667033731937, 20.79166006296873, 99.37502909451723, 20.875013433396816, 20.74998337775469, 18.79164483398199, 21.12501533702016 ], "command_buffer_latency_us_p50": 21.083338651806116, "command_buffer_latency_us_p90": 110.68335734307767 }, "throughput": [ { "compute_ns": 1059709.0, "compute_span_ns": 1059709.0, "command_buffer_ns": 1059750.0368021429, "calibration_cpu_ns": 1764250.0, "calibration_gpu_ticks": 1764250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1282875.0, "compute_span_ns": 1282875.0, "command_buffer_ns": 1282874.9604523182, "calibration_cpu_ns": 2678709.0, "calibration_gpu_ticks": 2678709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1034667.0, "compute_span_ns": 1034667.0, "command_buffer_ns": 1034666.6676923633, "calibration_cpu_ns": 1749958.0, "calibration_gpu_ticks": 1749958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1410875.0, "compute_span_ns": 1410875.0, "command_buffer_ns": 1410875.003784895, "calibration_cpu_ns": 2765375.0, "calibration_gpu_ticks": 2765375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1033208.0, "compute_span_ns": 1033208.0, "command_buffer_ns": 1033166.6562706232, "calibration_cpu_ns": 1765083.0, "calibration_gpu_ticks": 1765083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1286833.0, "compute_span_ns": 1286833.0, "command_buffer_ns": 1286791.6957475245, "calibration_cpu_ns": 2644166.0, "calibration_gpu_ticks": 2644166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1044709.0, "compute_span_ns": 1044709.0, "command_buffer_ns": 1044749.9807924032, "calibration_cpu_ns": 1673750.0, "calibration_gpu_ticks": 1673750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1262833.0, "compute_span_ns": 1262833.0, "command_buffer_ns": 1262833.3643078804, "calibration_cpu_ns": 2790833.0, "calibration_gpu_ticks": 2790833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1044042.0, "compute_span_ns": 1044042.0, "command_buffer_ns": 1044083.3284519613, "calibration_cpu_ns": 1814792.0, "calibration_gpu_ticks": 1814792.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 22583.0, "compute_span_ns": 22583.0, "command_buffer_ns": 22583.35007354617, "calibration_cpu_ns": 965959.0, "calibration_gpu_ticks": 965959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18875.0, "compute_span_ns": 18875.0, "command_buffer_ns": 18874.998204410076, "calibration_cpu_ns": 439500.0, "calibration_gpu_ticks": 439500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 19708.0, "compute_span_ns": 19708.0, "command_buffer_ns": 19666.622392833233, "calibration_cpu_ns": 761125.0, "calibration_gpu_ticks": 761125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18959.0, "compute_span_ns": 18959.0, "command_buffer_ns": 19000.028260052204, "calibration_cpu_ns": 638250.0, "calibration_gpu_ticks": 638250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20333.0, "compute_span_ns": 20333.0, "command_buffer_ns": 20333.33294093609, "calibration_cpu_ns": 868834.0, "calibration_gpu_ticks": 868834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22417.0, "compute_span_ns": 22417.0, "command_buffer_ns": 22458.378225564957, "calibration_cpu_ns": 449750.0, "calibration_gpu_ticks": 449750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 93959.0, "compute_span_ns": 93959.0, "command_buffer_ns": 94000.01727044582, "calibration_cpu_ns": 1178417.0, "calibration_gpu_ticks": 1178417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21000.0, "compute_span_ns": 21000.0, "command_buffer_ns": 21000.043489038944, "calibration_cpu_ns": 843250.0, "calibration_gpu_ticks": 843250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21042.0, "compute_span_ns": 21042.0, "command_buffer_ns": 21083.338651806116, "calibration_cpu_ns": 457875.0, "calibration_gpu_ticks": 457875.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 16.557953125, 20.044921875, 16.166671875, 22.044921875, 16.143875, 20.106765625, 16.323578125, 19.731765625, 16.31315625 ], "compute_throughput_us_p50": 16.557953125, "compute_throughput_us_p90": 20.494396875, "command_buffer_throughput_us": [ 16.558594325033482, 20.04492125706747, 16.166666682693176, 22.044921934138983, 16.143229004228488, 20.10612024605507, 16.3242184498813, 19.73177131731063, 16.313802007061895 ], "command_buffer_throughput_us_p50": 16.558594325033482, "command_buffer_throughput_us_p90": 20.493880583671853, "compute_latency_us": [ 22.583, 18.875, 19.708, 18.959, 20.333, 22.417, 93.959, 21.0, 21.042 ], "compute_latency_us_p50": 21.0, "compute_latency_us_p90": 36.85820000000001, "command_buffer_latency_us": [ 22.58335007354617, 18.874998204410076, 19.666622392833233, 19.000028260052204, 20.33333294093609, 22.458378225564957, 94.00001727044582, 21.000043489038944, 21.083338651806116 ], "command_buffer_latency_us_p50": 21.000043489038944, "command_buffer_latency_us_p90": 36.866683512926116, "counter_control_throughput_ratio": 0.7962504100061157, "counter_control_latency_ratio": 0.9138293047553457 }, "download_ms": 0.364417, "allocation_upload_ms": 0.795334, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-a62lcxbm/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b5a32df1046f38df4f8e9cbba5277df92f4beb4d4f9631ac4564ef5cda134264", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.657, "capture_ms": 0.065416, "compile_ms": 56.442125, "allocation_upload_ms": 3.4105, "cold_call_ms": 3.979, "warmup_ms": 200.036209, "download_ms": 0.952083, "repetitions": 1181, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 21.3798687553, 21.1161803556, 21.2635478408, 21.2423090601, 21.8577832345, 22.0325292125, 21.3736951736, 21.4887806943, 21.4624961897 ], "latency_us": [ 316.25, 2343.375, 569.292, 337.583, 482.167, 253.666, 305.375, 260.833, 282.042 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 68, "normalized_kernel_cost": 272, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1328083, "compute_span_ns": 1328083, "calibration_cpu_ns": 2678958, "calibration_gpu_ticks": 2678958, "compute_passes": 1, "command_buffer_ns": 1328125.01397, "command_buffers": 2 }, { "compute_ns": 935750, "compute_span_ns": 935750, "calibration_cpu_ns": 1585750, "calibration_gpu_ticks": 1585750, "compute_passes": 1, "command_buffer_ns": 935833.31909, "command_buffers": 2 }, { "compute_ns": 1356500, "compute_span_ns": 1356500, "calibration_cpu_ns": 2638292, "calibration_gpu_ticks": 2638292, "compute_passes": 1, "command_buffer_ns": 1356625.05636, "command_buffers": 2 }, { "compute_ns": 973583, "compute_span_ns": 973583, "calibration_cpu_ns": 1532750, "calibration_gpu_ticks": 1532750, "compute_passes": 1, "command_buffer_ns": 973666.66887, "command_buffers": 2 }, { "compute_ns": 1290917, "compute_span_ns": 1290917, "calibration_cpu_ns": 2643000, "calibration_gpu_ticks": 2643000, "compute_passes": 1, "command_buffer_ns": 1291083.28838, "command_buffers": 2 }, { "compute_ns": 935708, "compute_span_ns": 935708, "calibration_cpu_ns": 1521833, "calibration_gpu_ticks": 1521833, "compute_passes": 1, "command_buffer_ns": 935791.700613, "command_buffers": 2 }, { "compute_ns": 1362042, "compute_span_ns": 1362042, "calibration_cpu_ns": 2446584, "calibration_gpu_ticks": 2446584, "compute_passes": 1, "command_buffer_ns": 1362125.04003, "command_buffers": 2 }, { "compute_ns": 924750, "compute_span_ns": 924750, "calibration_cpu_ns": 1571500, "calibration_gpu_ticks": 1571500, "compute_passes": 1, "command_buffer_ns": 924791.675061, "command_buffers": 2 }, { "compute_ns": 1390625, "compute_span_ns": 1390625, "calibration_cpu_ns": 2729833, "calibration_gpu_ticks": 2729833, "compute_passes": 1, "command_buffer_ns": 1390749.93785, "command_buffers": 2 } ], "latency": [ { "compute_ns": 21542, "compute_span_ns": 21542, "calibration_cpu_ns": 1105458, "calibration_gpu_ticks": 1105458, "compute_passes": 1, "command_buffer_ns": 21708.3725147, "command_buffers": 2 }, { "compute_ns": 19125, "compute_span_ns": 19125, "calibration_cpu_ns": 789208, "calibration_gpu_ticks": 789208, "compute_passes": 1, "command_buffer_ns": 19166.6767932, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 422167, "calibration_gpu_ticks": 422167, "compute_passes": 1, "command_buffer_ns": 18500.0244528, "command_buffers": 2 }, { "compute_ns": 85083, "compute_span_ns": 85083, "calibration_cpu_ns": 1441917, "calibration_gpu_ticks": 1441917, "compute_passes": 1, "command_buffer_ns": 85083.3603181, "command_buffers": 2 }, { "compute_ns": 18416, "compute_span_ns": 18416, "calibration_cpu_ns": 541084, "calibration_gpu_ticks": 541084, "compute_passes": 1, "command_buffer_ns": 18374.9943972, "command_buffers": 2 }, { "compute_ns": 42292, "compute_span_ns": 42292, "calibration_cpu_ns": 1437750, "calibration_gpu_ticks": 1437750, "compute_passes": 1, "command_buffer_ns": 42333.3258368, "command_buffers": 2 }, { "compute_ns": 19042, "compute_span_ns": 19042, "calibration_cpu_ns": 479000, "calibration_gpu_ticks": 479000, "compute_passes": 1, "command_buffer_ns": 19125.000108, "command_buffers": 2 }, { "compute_ns": 63667, "compute_span_ns": 63667, "calibration_cpu_ns": 1593041, "calibration_gpu_ticks": 1593041, "compute_passes": 1, "command_buffer_ns": 63791.6964479, "command_buffers": 2 }, { "compute_ns": 18417, "compute_span_ns": 18417, "calibration_cpu_ns": 463459, "calibration_gpu_ticks": 463459, "compute_passes": 1, "command_buffer_ns": 18458.3477676, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1233333.3143, "command_buffers": 2 }, { "command_buffer_ns": 1445916.65594, "command_buffers": 2 }, { "command_buffer_ns": 939041.72536, "command_buffers": 2 }, { "command_buffer_ns": 1179624.93096, "command_buffers": 2 }, { "command_buffer_ns": 938250.042964, "command_buffers": 2 }, { "command_buffer_ns": 1355958.34581, "command_buffers": 2 }, { "command_buffer_ns": 935750.023928, "command_buffers": 2 }, { "command_buffer_ns": 1421249.99547, "command_buffers": 2 }, { "command_buffer_ns": 946750.049479, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 17291.6916199, "command_buffers": 2 }, { "command_buffer_ns": 18666.6147783, "command_buffers": 2 }, { "command_buffer_ns": 18250.0225492, "command_buffers": 2 }, { "command_buffer_ns": 31958.3341479, "command_buffers": 2 }, { "command_buffer_ns": 16875.0411831, "command_buffers": 2 }, { "command_buffer_ns": 19708.299078, "command_buffers": 2 }, { "command_buffer_ns": 16583.2461789, "command_buffers": 2 }, { "command_buffer_ns": 20208.361093, "command_buffers": 2 }, { "command_buffer_ns": 16250.0073202, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 19.2708330359375, 22.5924477490625, 14.67252695875, 18.43163954625, 14.6601569213125, 21.18684915328125, 14.621094123875, 22.20703117921875, 14.792969523109376 ], "command_buffer_throughput_us_p50": 18.43163954625, "command_buffer_throughput_us_p90": 22.2841144931875, "command_buffer_latency_us": [ 17.2916916199, 18.6666147783, 18.2500225492, 31.9583341479, 16.8750411831, 19.708299078, 16.5832461789, 20.208361093, 16.2500073202 ], "command_buffer_latency_us_p50": 18.2500225492, "command_buffer_latency_us_p90": 22.558355703980002 }, "compute_throughput_us": [ 20.751296875, 14.62109375, 21.1953125, 15.212234375, 20.170578125, 14.6204375, 21.28190625, 14.44921875, 21.728515625 ], "compute_throughput_us_p50": 20.170578125, "compute_throughput_us_p90": 21.371228125, "command_buffer_throughput_us": [ 20.75195334328125, 14.62239561078125, 21.197266505625002, 15.213541701093751, 20.1731763809375, 14.621745322078125, 21.28320375046875, 14.449869922828126, 21.730467778906252 ], "command_buffer_throughput_us_p50": 20.1731763809375, "command_buffer_throughput_us_p90": 21.37265655615625, "compute_latency_us": [ 21.542, 19.125, 18.5, 85.083, 18.416, 42.292, 19.042, 63.667, 18.417 ], "compute_latency_us_p50": 19.125, "compute_latency_us_p90": 67.95020000000001, "command_buffer_latency_us": [ 21.7083725147, 19.1666767932, 18.5000244528, 85.0833603181, 18.374994397200002, 42.3333258368, 19.125000108000002, 63.7916964479, 18.4583477676 ], "command_buffer_latency_us_p50": 19.1666767932, "command_buffer_latency_us_p90": 68.05002922194001, "counter_control_throughput_ratio": 1.0768581360536755, "counter_control_latency_ratio": 1.1532723992443679 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 21.3798687553, "throughput_us_p90": 21.8927324301, "latency_us_p50": 316.25, "latency_us_p90": 924.1086000000004 }, "slowdown": 0.7797901687681722, "device_compute_slowdown": 1.2181806514807365, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-okhrvgao/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "232522da57db77c64b377ec2facdc2e1df33cd57064a4297b89d6582fd4ad63b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.381041, "capture_ms": 0.063292, "compile_ms": 57.025917, "allocation_upload_ms": 7.65775, "cold_call_ms": 6.199125, "warmup_ms": 200.646916, "download_ms": 2.975083, "repetitions": 309, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 90.9286666667, 86.7580906149, 91.0985695793, 90.1441456311, 87.2042880259, 90.8408867314, 90.5246763754, 87.0440938511, 87.8704142395 ], "latency_us": [ 2411.833, 455.791, 462.916, 273.5, 356.333, 292.666, 300.083, 266.334, 312.084 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 104, "normalized_kernel_cost": 1248, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4206167, "compute_span_ns": 4206167, "calibration_cpu_ns": 5313542, "calibration_gpu_ticks": 5313542, "compute_passes": 1, "command_buffer_ns": 4206250.01425, "command_buffers": 2 }, { "compute_ns": 4201666, "compute_span_ns": 4201666, "calibration_cpu_ns": 5403000, "calibration_gpu_ticks": 5403000, "compute_passes": 1, "command_buffer_ns": 4201666.62661, "command_buffers": 2 }, { "compute_ns": 6347583, "compute_span_ns": 6347583, "calibration_cpu_ns": 8921042, "calibration_gpu_ticks": 8921042, "compute_passes": 1, "command_buffer_ns": 6347708.28765, "command_buffers": 2 }, { "compute_ns": 3826792, "compute_span_ns": 3826792, "calibration_cpu_ns": 4698584, "calibration_gpu_ticks": 4698584, "compute_passes": 1, "command_buffer_ns": 3826875.00678, "command_buffers": 2 }, { "compute_ns": 4262042, "compute_span_ns": 4262042, "calibration_cpu_ns": 5335334, "calibration_gpu_ticks": 5335334, "compute_passes": 1, "command_buffer_ns": 4262125.00315, "command_buffers": 2 }, { "compute_ns": 4313125, "compute_span_ns": 4313125, "calibration_cpu_ns": 5557958, "calibration_gpu_ticks": 5557958, "compute_passes": 1, "command_buffer_ns": 4313250.01409, "command_buffers": 2 }, { "compute_ns": 4312459, "compute_span_ns": 4312459, "calibration_cpu_ns": 5276583, "calibration_gpu_ticks": 5276583, "compute_passes": 1, "command_buffer_ns": 4312500.00838, "command_buffers": 2 }, { "compute_ns": 4179958, "compute_span_ns": 4179958, "calibration_cpu_ns": 5376834, "calibration_gpu_ticks": 5376834, "compute_passes": 1, "command_buffer_ns": 4180041.66568, "command_buffers": 2 }, { "compute_ns": 6493667, "compute_span_ns": 6493667, "calibration_cpu_ns": 8285916, "calibration_gpu_ticks": 8285916, "compute_passes": 1, "command_buffer_ns": 6493750.02831, "command_buffers": 2 } ], "latency": [ { "compute_ns": 119583, "compute_span_ns": 119583, "calibration_cpu_ns": 1111959, "calibration_gpu_ticks": 1111959, "compute_passes": 1, "command_buffer_ns": 119541.655295, "command_buffers": 2 }, { "compute_ns": 62166, "compute_span_ns": 62166, "calibration_cpu_ns": 512375, "calibration_gpu_ticks": 512375, "compute_passes": 1, "command_buffer_ns": 62249.9501333, "command_buffers": 2 }, { "compute_ns": 61375, "compute_span_ns": 61375, "calibration_cpu_ns": 473417, "calibration_gpu_ticks": 473417, "compute_passes": 1, "command_buffer_ns": 61375.0307821, "command_buffers": 2 }, { "compute_ns": 63750, "compute_span_ns": 63750, "calibration_cpu_ns": 486333, "calibration_gpu_ticks": 486333, "compute_passes": 1, "command_buffer_ns": 63749.961555, "command_buffers": 2 }, { "compute_ns": 116666, "compute_span_ns": 116666, "calibration_cpu_ns": 1192125, "calibration_gpu_ticks": 1192125, "compute_passes": 1, "command_buffer_ns": 116666.720714, "command_buffers": 2 }, { "compute_ns": 111959, "compute_span_ns": 111959, "calibration_cpu_ns": 830458, "calibration_gpu_ticks": 830458, "compute_passes": 1, "command_buffer_ns": 111999.979708, "command_buffers": 2 }, { "compute_ns": 110750, "compute_span_ns": 110750, "calibration_cpu_ns": 1340958, "calibration_gpu_ticks": 1340958, "compute_passes": 1, "command_buffer_ns": 110833.323561, "command_buffers": 2 }, { "compute_ns": 80000, "compute_span_ns": 80000, "calibration_cpu_ns": 578042, "calibration_gpu_ticks": 578042, "compute_passes": 1, "command_buffer_ns": 80083.3222456, "command_buffers": 2 }, { "compute_ns": 68542, "compute_span_ns": 68542, "calibration_cpu_ns": 509750, "calibration_gpu_ticks": 509750, "compute_passes": 1, "command_buffer_ns": 68624.9695718, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4086416.68037, "command_buffers": 2 }, { "command_buffer_ns": 4430249.97367, "command_buffers": 2 }, { "command_buffer_ns": 4039124.99314, "command_buffers": 2 }, { "command_buffer_ns": 3872708.35927, "command_buffers": 2 }, { "command_buffer_ns": 4029916.6576, "command_buffers": 2 }, { "command_buffer_ns": 4443541.63483, "command_buffers": 2 }, { "command_buffer_ns": 4323541.65241, "command_buffers": 2 }, { "command_buffer_ns": 6569916.61526, "command_buffers": 2 }, { "command_buffer_ns": 3787249.96699, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 69833.3024047, "command_buffers": 2 }, { "command_buffer_ns": 252791.680396, "command_buffers": 2 }, { "command_buffer_ns": 108416.716103, "command_buffers": 2 }, { "command_buffer_ns": 117291.69637, "command_buffers": 2 }, { "command_buffer_ns": 63374.9878034, "command_buffers": 2 }, { "command_buffer_ns": 120166.689157, "command_buffers": 2 }, { "command_buffer_ns": 88125.0016391, "command_buffers": 2 }, { "command_buffer_ns": 61791.6812189, "command_buffers": 2 }, { "command_buffer_ns": 131833.308842, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 63.85026063078125, 69.22265583859375, 63.1113280178125, 60.51106811359375, 62.967447775000004, 69.43033804421874, 67.55533831890624, 102.6549471134375, 59.175780734218755 ], "command_buffer_throughput_us_p50": 63.85026063078125, "command_buffer_throughput_us_p90": 76.0752598580625, "command_buffer_latency_us": [ 69.8333024047, 252.791680396, 108.416716103, 117.29169637000001, 63.374987803399996, 120.16668915700001, 88.12500163909999, 61.791681218899996, 131.833308842 ], "command_buffer_latency_us_p50": 108.416716103, "command_buffer_latency_us_p90": 156.02498315280002 }, "compute_throughput_us": [ 65.721359375, 65.65103125, 99.180984375, 59.793625, 66.59440625, 67.392578125, 67.382171875, 65.31184375, 101.463546875 ], "compute_throughput_us_p50": 66.59440625, "compute_throughput_us_p90": 99.637496875, "command_buffer_throughput_us": [ 65.72265647265625, 65.65104104078125, 99.18294199453125, 59.7949219809375, 66.59570317421876, 67.39453147015625, 67.3828126309375, 65.31315102625, 101.46484419234375 ], "command_buffer_throughput_us_p50": 66.59570317421876, "command_buffer_throughput_us_p90": 99.63932243409374, "compute_latency_us": [ 119.583, 62.166, 61.375, 63.75, 116.666, 111.959, 110.75, 80.0, 68.542 ], "compute_latency_us_p50": 80.0, "compute_latency_us_p90": 117.2494, "command_buffer_latency_us": [ 119.541655295, 62.249950133300004, 61.375030782100005, 63.749961555, 116.666720714, 111.999979708, 110.833323561, 80.08332224559999, 68.6249695718 ], "command_buffer_latency_us_p50": 80.08332224559999, "command_buffer_latency_us_p90": 117.2417076302, "counter_control_throughput_ratio": 0.9974461575907693, "counter_control_latency_ratio": 0.932038491646133 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 90.1441456311, "throughput_us_p90": 90.96264724922, "latency_us_p50": 312.084, "latency_us_p90": 852.6994000000004 }, "torch": { "cold_call_ms": 10.628375, "warmup_ms": 200.270625, "repetitions": 269, "throughput_us": [ 120.81707063197025, 121.27663940520445, 120.63150557620818, 120.28609293680299, 112.3130408921933, 115.52571375464683, 120.97877695167288, 120.31505576208178, 120.67224163568774 ], "latency_us": [ 447.916, 398.625, 355.66700000000003, 331.5, 322.20799999999997, 329.25, 362.791, 325.25, 315.125 ], "throughput_us_p50": 120.63150557620818, "throughput_us_p90": 121.0383494423792, "latency_us_p50": 331.5, "latency_us_p90": 408.4832, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5195499.979890883, "command_buffers": 1 }, { "command_buffer_ns": 5021708.377171308, "command_buffers": 1 }, { "command_buffer_ns": 5403625.022154301, "command_buffers": 1 }, { "command_buffer_ns": 5309833.330102265, "command_buffers": 1 }, { "command_buffer_ns": 5122291.622683406, "command_buffers": 1 }, { "command_buffer_ns": 5168750.009033829, "command_buffers": 1 }, { "command_buffer_ns": 5075541.674159467, "command_buffers": 1 }, { "command_buffer_ns": 5003666.679840535, "command_buffers": 1 }, { "command_buffer_ns": 7759624.975733459, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 219666.6900999844, "command_buffers": 1 }, { "command_buffer_ns": 216874.9924749136, "command_buffers": 1 }, { "command_buffer_ns": 95749.97238814831, "command_buffers": 1 }, { "command_buffer_ns": 173583.33570882678, "command_buffers": 1 }, { "command_buffer_ns": 87374.9959282577, "command_buffers": 1 }, { "command_buffer_ns": 243375.01963600516, "command_buffers": 1 }, { "command_buffer_ns": 110124.99453499913, "command_buffers": 1 }, { "command_buffer_ns": 113833.34640413523, "command_buffers": 1 }, { "command_buffer_ns": 122500.00145286322, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 81.17968718579505, 78.46419339330168, 84.43164097116096, 82.96614578284789, 80.03580660442822, 80.76171889115358, 79.30533865874168, 78.18229187250836, 121.2441402458353 ], "command_buffer_throughput_us_p50": 80.76171889115358, "command_buffer_throughput_us_p90": 91.79414082609583, "command_buffer_latency_us": [ 219.6666900999844, 216.8749924749136, 95.74997238814831, 173.58333570882678, 87.3749959282577, 243.37501963600516, 110.12499453499913, 113.83334640413523, 122.50000145286322 ], "command_buffer_latency_us_p50": 122.50000145286322, "command_buffer_latency_us_p90": 224.40835600718856 }, "throughput": [ { "compute_ns": 5359875.0, "compute_span_ns": 5359875.0, "command_buffer_ns": 5359874.980058521, "calibration_cpu_ns": 6745500.0, "calibration_gpu_ticks": 6745500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5216208.0, "compute_span_ns": 5216208.0, "command_buffer_ns": 5216208.3447910845, "calibration_cpu_ns": 6532000.0, "calibration_gpu_ticks": 6532000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5219292.0, "compute_span_ns": 5219292.0, "command_buffer_ns": 5219333.339482546, "calibration_cpu_ns": 6600042.0, "calibration_gpu_ticks": 6600042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5035250.0, "compute_span_ns": 5035250.0, "command_buffer_ns": 5035249.9820292, "calibration_cpu_ns": 7864500.0, "calibration_gpu_ticks": 7864500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5015458.0, "compute_span_ns": 5015458.0, "command_buffer_ns": 5015416.65289551, "calibration_cpu_ns": 7042875.0, "calibration_gpu_ticks": 7042875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5077792.0, "compute_span_ns": 5077792.0, "command_buffer_ns": 5077791.691292077, "calibration_cpu_ns": 7366375.0, "calibration_gpu_ticks": 7366375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5060708.0, "compute_span_ns": 5060708.0, "command_buffer_ns": 5060666.64820537, "calibration_cpu_ns": 6855541.0, "calibration_gpu_ticks": 6855541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5053000.0, "compute_span_ns": 5053000.0, "command_buffer_ns": 5053000.000771135, "calibration_cpu_ns": 7307708.0, "calibration_gpu_ticks": 7307708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5017625.0, "compute_span_ns": 5017625.0, "command_buffer_ns": 5017624.993342906, "calibration_cpu_ns": 7034875.0, "calibration_gpu_ticks": 7034875.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 84666.0, "compute_span_ns": 84666.0, "command_buffer_ns": 84625.03319606185, "calibration_cpu_ns": 707291.0, "calibration_gpu_ticks": 707291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80792.0, "compute_span_ns": 80792.0, "command_buffer_ns": 80833.32795649767, "calibration_cpu_ns": 667292.0, "calibration_gpu_ticks": 667292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81083.0, "compute_span_ns": 81083.0, "command_buffer_ns": 81041.65317490697, "calibration_cpu_ns": 618334.0, "calibration_gpu_ticks": 618334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81583.0, "compute_span_ns": 81583.0, "command_buffer_ns": 81541.65698215365, "calibration_cpu_ns": 512208.0, "calibration_gpu_ticks": 512208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80125.0, "compute_span_ns": 80125.0, "command_buffer_ns": 80124.99893084168, "calibration_cpu_ns": 617500.0, "calibration_gpu_ticks": 617500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 120750.0, "compute_span_ns": 120750.0, "command_buffer_ns": 120749.98812749982, "calibration_cpu_ns": 950417.0, "calibration_gpu_ticks": 950417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84542.0, "compute_span_ns": 84542.0, "command_buffer_ns": 84541.67982563376, "calibration_cpu_ns": 599625.0, "calibration_gpu_ticks": 599625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81250.0, "compute_span_ns": 81250.0, "command_buffer_ns": 81250.03660097718, "calibration_cpu_ns": 1119875.0, "calibration_gpu_ticks": 1119875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 122959.0, "compute_span_ns": 122959.0, "command_buffer_ns": 123000.0052601099, "calibration_cpu_ns": 1306250.0, "calibration_gpu_ticks": 1306250.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 83.748046875, 81.50325, 81.5514375, 78.67578125, 78.36653125, 79.3405, 79.0735625, 78.953125, 78.400390625 ], "compute_throughput_us_p50": 79.0735625, "compute_throughput_us_p90": 81.99075937500001, "command_buffer_throughput_us": [ 83.74804656341439, 81.5032553873607, 81.55208342941478, 78.67578096920624, 78.36588520149235, 79.34049517643871, 79.0729163782089, 78.95312501204899, 78.40039052098291 ], "command_buffer_throughput_us_p50": 79.0729163782089, "command_buffer_throughput_us_p90": 81.9912760562147, "compute_latency_us": [ 84.666, 80.792, 81.083, 81.583, 80.125, 120.75, 84.542, 81.25, 122.959 ], "compute_latency_us_p50": 81.583, "compute_latency_us_p90": 121.1918, "command_buffer_latency_us": [ 84.62503319606185, 80.83332795649767, 81.04165317490697, 81.54165698215365, 80.12499893084168, 120.74998812749982, 84.54167982563376, 81.25003660097718, 123.0000052601099 ], "command_buffer_latency_us_p50": 81.54165698215365, "command_buffer_latency_us_p90": 121.19999155402184, "counter_control_throughput_ratio": 0.9824022602016393, "counter_control_latency_ratio": 0.7137630507015088 }, "download_ms": 0.686, "allocation_upload_ms": 18.107625, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7472686774530225, "device_compute_slowdown": 0.8421829514763548, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.387458, "warmup_ms": 200.510709, "repetitions": 206, "throughput_us": [ 119.72128155339806, 117.66100485436894, 109.85133495145631, 115.09466019417475, 121.90533980582525, 120.96783980582525, 121.90473300970874, 120.36084466019418, 122.662213592233 ], "latency_us": [ 424.58299999999997, 2472.042, 876.125, 547.125, 280.0, 384.625, 311.0, 318.75, 317.75 ], "throughput_us_p50": 120.36084466019418, "throughput_us_p90": 122.0567145631068, "latency_us_p50": 384.625, "latency_us_p90": 1195.3084000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5284458.340611309, "command_buffers": 1 }, { "command_buffer_ns": 5159749.99871105, "command_buffers": 1 }, { "command_buffer_ns": 5325749.982148409, "command_buffers": 1 }, { "command_buffer_ns": 5115000.007208437, "command_buffers": 1 }, { "command_buffer_ns": 5245875.043328851, "command_buffers": 1 }, { "command_buffer_ns": 5102499.9702349305, "command_buffers": 1 }, { "command_buffer_ns": 5152666.650246829, "command_buffers": 1 }, { "command_buffer_ns": 7571333.320811391, "command_buffers": 1 }, { "command_buffer_ns": 7505499.990656972, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 117124.98962879181, "command_buffers": 1 }, { "command_buffer_ns": 83499.99552592635, "command_buffers": 1 }, { "command_buffer_ns": 118333.32246169448, "command_buffers": 1 }, { "command_buffer_ns": 80375.00083446503, "command_buffers": 1 }, { "command_buffer_ns": 117291.69636964798, "command_buffers": 1 }, { "command_buffer_ns": 80999.97648969293, "command_buffers": 1 }, { "command_buffer_ns": 116875.04593282938, "command_buffers": 1 }, { "command_buffer_ns": 117000.0177808106, "command_buffers": 1 }, { "command_buffer_ns": 114166.64347052574, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 82.5696615720517, 80.62109372986015, 83.21484347106889, 79.92187511263182, 81.9667975520133, 79.72656203492079, 80.5104164101067, 118.30208313767798, 117.27343735401519 ], "command_buffer_throughput_us_p50": 81.9667975520133, "command_buffer_throughput_us_p90": 117.47916651074775, "command_buffer_latency_us": [ 117.12498962879181, 83.49999552592635, 118.33332246169448, 80.37500083446503, 117.29169636964798, 80.99997648969293, 116.87504593282938, 117.0000177808106, 114.16664347052574 ], "command_buffer_latency_us_p50": 116.87504593282938, "command_buffer_latency_us_p90": 117.50002158805728 }, "throughput": [ { "compute_ns": 5124125.0, "compute_span_ns": 5124125.0, "command_buffer_ns": 5124124.989379197, "calibration_cpu_ns": 6726625.0, "calibration_gpu_ticks": 6726625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5276500.0, "compute_span_ns": 5276500.0, "command_buffer_ns": 5276500.014588237, "calibration_cpu_ns": 7237250.0, "calibration_gpu_ticks": 7237250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5156250.0, "compute_span_ns": 5156250.0, "command_buffer_ns": 5156249.972060323, "calibration_cpu_ns": 7359916.0, "calibration_gpu_ticks": 7359916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5035458.0, "compute_span_ns": 5035458.0, "command_buffer_ns": 5035416.688770056, "calibration_cpu_ns": 7196875.0, "calibration_gpu_ticks": 7196875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4970375.0, "compute_span_ns": 4970375.0, "command_buffer_ns": 4970374.98280406, "calibration_cpu_ns": 7754417.0, "calibration_gpu_ticks": 7754417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4983625.0, "compute_span_ns": 4983625.0, "command_buffer_ns": 4983624.967280775, "calibration_cpu_ns": 7323542.0, "calibration_gpu_ticks": 7323542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5246458.0, "compute_span_ns": 5246458.0, "command_buffer_ns": 5246458.342298865, "calibration_cpu_ns": 6513208.0, "calibration_gpu_ticks": 6513208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5159958.0, "compute_span_ns": 5159958.0, "command_buffer_ns": 5159916.647244245, "calibration_cpu_ns": 6634542.0, "calibration_gpu_ticks": 6634542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5203833.0, "compute_span_ns": 5203833.0, "command_buffer_ns": 5203833.33787322, "calibration_cpu_ns": 7333584.0, "calibration_gpu_ticks": 7333584.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 81791.0, "compute_span_ns": 81791.0, "command_buffer_ns": 81749.98220056295, "calibration_cpu_ns": 841417.0, "calibration_gpu_ticks": 841417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 114292.0, "compute_span_ns": 114292.0, "command_buffer_ns": 114291.67352616787, "calibration_cpu_ns": 803042.0, "calibration_gpu_ticks": 803042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80541.0, "compute_span_ns": 80541.0, "command_buffer_ns": 80499.97268244624, "calibration_cpu_ns": 633375.0, "calibration_gpu_ticks": 633375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 127750.0, "compute_span_ns": 127750.0, "command_buffer_ns": 127750.04142895341, "calibration_cpu_ns": 1204875.0, "calibration_gpu_ticks": 1204875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 119500.0, "compute_span_ns": 119500.0, "command_buffer_ns": 119500.03681704402, "calibration_cpu_ns": 1075333.0, "calibration_gpu_ticks": 1075333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 115083.0, "compute_span_ns": 115083.0, "command_buffer_ns": 115083.35592225194, "calibration_cpu_ns": 793667.0, "calibration_gpu_ticks": 793667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81750.0, "compute_span_ns": 81750.0, "command_buffer_ns": 81749.98220056295, "calibration_cpu_ns": 518834.0, "calibration_gpu_ticks": 518834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80917.0, "compute_span_ns": 80917.0, "command_buffer_ns": 80958.29980447888, "calibration_cpu_ns": 503000.0, "calibration_gpu_ticks": 503000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81417.0, "compute_span_ns": 81417.0, "command_buffer_ns": 81416.68513417244, "calibration_cpu_ns": 665958.0, "calibration_gpu_ticks": 665958.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 80.064453125, 82.4453125, 80.56640625, 78.67903125, 77.662109375, 77.869140625, 81.97590625, 80.62434375, 81.309890625 ], "compute_throughput_us_p50": 80.56640625, "compute_throughput_us_p90": 82.06978749999999, "command_buffer_throughput_us": [ 80.06445295904996, 82.4453127279412, 80.56640581344254, 78.67838576203212, 77.66210910631344, 77.86914011376211, 81.97591159841977, 80.62369761319133, 81.30989590426907 ], "command_buffer_throughput_us_p50": 80.56640581344254, "command_buffer_throughput_us_p90": 82.06979182432406, "compute_latency_us": [ 81.791, 114.292, 80.541, 127.75, 119.5, 115.083, 81.75, 80.917, 81.417 ], "compute_latency_us_p50": 81.791, "compute_latency_us_p90": 121.15, "command_buffer_latency_us": [ 81.74998220056295, 114.29167352616787, 80.49997268244624, 127.75004142895341, 119.50003681704402, 115.08335592225194, 81.74998220056295, 80.95829980447888, 81.41668513417244 ], "command_buffer_latency_us_p50": 81.74998220056295, "command_buffer_latency_us_p90": 121.1500377394259, "counter_control_throughput_ratio": 0.9696594540257907, "counter_control_latency_ratio": 0.7131389927846279 }, "download_ms": 0.689208, "allocation_upload_ms": 1.197667, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-mk1m5c_5/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "c9fd11968a5edbf77e894438f42e4bd465ad146629716a6f08eacac9ab82063f", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.440167, "capture_ms": 0.077708, "compile_ms": 56.398291, "allocation_upload_ms": 9.104458, "cold_call_ms": 9.796667, "warmup_ms": 200.538375, "download_ms": 3.475291, "repetitions": 309, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 93.998381877, 100.738809061, 91.8594919094, 100.939184466, 95.8807993528, 95.8948220065, 101.055957929, 92.5316893204, 100.418016181 ], "latency_us": [ 417.541, 318.5, 279.042, 288, 278.625, 342.167, 2067.917, 672.792, 422.417 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 96, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4669542, "compute_span_ns": 4669542, "calibration_cpu_ns": 5805750, "calibration_gpu_ticks": 5805750, "compute_passes": 1, "command_buffer_ns": 4669708.31621, "command_buffers": 2 }, { "compute_ns": 4763375, "compute_span_ns": 4763375, "calibration_cpu_ns": 5744583, "calibration_gpu_ticks": 5744583, "compute_passes": 1, "command_buffer_ns": 4763500.00827, "command_buffers": 2 }, { "compute_ns": 4665208, "compute_span_ns": 4665208, "calibration_cpu_ns": 5749916, "calibration_gpu_ticks": 5749916, "compute_passes": 1, "command_buffer_ns": 4665291.69353, "command_buffers": 2 }, { "compute_ns": 4509917, "compute_span_ns": 4509917, "calibration_cpu_ns": 5644833, "calibration_gpu_ticks": 5644833, "compute_passes": 1, "command_buffer_ns": 4510000.05705, "command_buffers": 2 }, { "compute_ns": 4607500, "compute_span_ns": 4607500, "calibration_cpu_ns": 5698083, "calibration_gpu_ticks": 5698083, "compute_passes": 1, "command_buffer_ns": 4607541.60304, "command_buffers": 2 }, { "compute_ns": 4768333, "compute_span_ns": 4768333, "calibration_cpu_ns": 5774417, "calibration_gpu_ticks": 5774417, "compute_passes": 1, "command_buffer_ns": 4768458.36965, "command_buffers": 2 }, { "compute_ns": 4773250, "compute_span_ns": 4773250, "calibration_cpu_ns": 5726708, "calibration_gpu_ticks": 5726708, "compute_passes": 1, "command_buffer_ns": 4773333.31946, "command_buffers": 2 }, { "compute_ns": 4702041, "compute_span_ns": 4702041, "calibration_cpu_ns": 5711916, "calibration_gpu_ticks": 5711916, "compute_passes": 1, "command_buffer_ns": 4702125.03569, "command_buffers": 2 }, { "compute_ns": 4728875, "compute_span_ns": 4728875, "calibration_cpu_ns": 5813291, "calibration_gpu_ticks": 5813291, "compute_passes": 1, "command_buffer_ns": 4728958.30171, "command_buffers": 2 } ], "latency": [ { "compute_ns": 117625, "compute_span_ns": 117625, "calibration_cpu_ns": 857458, "calibration_gpu_ticks": 857458, "compute_passes": 1, "command_buffer_ns": 117666.670121, "command_buffers": 2 }, { "compute_ns": 66708, "compute_span_ns": 66708, "calibration_cpu_ns": 587500, "calibration_gpu_ticks": 587500, "compute_passes": 1, "command_buffer_ns": 66749.9843985, "command_buffers": 2 }, { "compute_ns": 112416, "compute_span_ns": 112416, "calibration_cpu_ns": 755667, "calibration_gpu_ticks": 755667, "compute_passes": 1, "command_buffer_ns": 112500.041723, "command_buffers": 2 }, { "compute_ns": 67000, "compute_span_ns": 67000, "calibration_cpu_ns": 599875, "calibration_gpu_ticks": 599875, "compute_passes": 1, "command_buffer_ns": 67124.9581501, "command_buffers": 2 }, { "compute_ns": 100667, "compute_span_ns": 100667, "calibration_cpu_ns": 768417, "calibration_gpu_ticks": 768417, "compute_passes": 1, "command_buffer_ns": 100666.65709, "command_buffers": 2 }, { "compute_ns": 67792, "compute_span_ns": 67792, "calibration_cpu_ns": 461458, "calibration_gpu_ticks": 461458, "compute_passes": 1, "command_buffer_ns": 67791.6686982, "command_buffers": 2 }, { "compute_ns": 118208, "compute_span_ns": 118208, "calibration_cpu_ns": 737584, "calibration_gpu_ticks": 737584, "compute_passes": 1, "command_buffer_ns": 118250.027299, "command_buffers": 2 }, { "compute_ns": 67042, "compute_span_ns": 67042, "calibration_cpu_ns": 576584, "calibration_gpu_ticks": 576584, "compute_passes": 1, "command_buffer_ns": 67124.9581501, "command_buffers": 2 }, { "compute_ns": 123000, "compute_span_ns": 123000, "calibration_cpu_ns": 656541, "calibration_gpu_ticks": 656541, "compute_passes": 1, "command_buffer_ns": 123000.00526, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4535374.98834, "command_buffers": 2 }, { "command_buffer_ns": 4638500.04598, "command_buffers": 2 }, { "command_buffer_ns": 4416625.01544, "command_buffers": 2 }, { "command_buffer_ns": 4669833.28806, "command_buffers": 2 }, { "command_buffer_ns": 4426833.35859, "command_buffers": 2 }, { "command_buffer_ns": 4672958.28275, "command_buffers": 2 }, { "command_buffer_ns": 4706374.95164, "command_buffers": 2 }, { "command_buffer_ns": 4690291.70927, "command_buffers": 2 }, { "command_buffer_ns": 4663708.32874, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 365958.316252, "command_buffers": 2 }, { "command_buffer_ns": 68791.618105, "command_buffers": 2 }, { "command_buffer_ns": 357416.633051, "command_buffers": 2 }, { "command_buffer_ns": 68333.290983, "command_buffers": 2 }, { "command_buffer_ns": 228833.348956, "command_buffers": 2 }, { "command_buffer_ns": 66666.6310281, "command_buffers": 2 }, { "command_buffer_ns": 320249.99382, "command_buffers": 2 }, { "command_buffer_ns": 66041.6553728, "command_buffers": 2 }, { "command_buffer_ns": 326166.686136, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 70.8652341928125, 72.4765632184375, 69.00976586625, 72.9661451259375, 69.16927122796875, 73.01497316796875, 73.53710861937499, 73.28580795734376, 72.8704426365625 ], "command_buffer_throughput_us_p50": 72.8704426365625, "command_buffer_throughput_us_p90": 73.33606808975001, "command_buffer_latency_us": [ 365.958316252, 68.791618105, 357.416633051, 68.333290983, 228.833348956, 66.6666310281, 320.24999382, 66.0416553728, 326.16668613599995 ], "command_buffer_latency_us_p50": 228.833348956, "command_buffer_latency_us_p90": 359.1249696912 }, "compute_throughput_us": [ 72.96159375, 74.427734375, 72.893875, 70.467453125, 71.9921875, 74.505203125, 74.58203125, 73.469390625, 73.888671875 ], "compute_throughput_us_p50": 73.469390625, "compute_throughput_us_p90": 74.52056875, "command_buffer_throughput_us": [ 72.96419244078125, 74.42968762921875, 72.89518271140625, 70.46875089140624, 71.9928375475, 74.50716202578124, 74.5833331165625, 73.47070368265625, 73.88997346421876 ], "command_buffer_throughput_us_p50": 73.47070368265625, "command_buffer_throughput_us_p90": 74.5223962439375, "compute_latency_us": [ 117.625, 66.708, 112.416, 67.0, 100.667, 67.792, 118.208, 67.042, 123.0 ], "compute_latency_us_p50": 100.667, "compute_latency_us_p90": 119.1664, "command_buffer_latency_us": [ 117.66667012100001, 66.7499843985, 112.500041723, 67.12495815010001, 100.66665708999999, 67.7916686982, 118.250027299, 67.12495815010001, 123.00000526000001 ], "command_buffer_latency_us_p50": 100.66665708999999, "command_buffer_latency_us_p90": 119.20002289119999, "counter_control_throughput_ratio": 1.020436751436993, "counter_control_latency_ratio": 0.4399125282624612 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 95.8948220065, "throughput_us_p90": 100.9625391586, "latency_us_p50": 342.167, "latency_us_p90": 951.8170000000002 }, "slowdown": 0.7967277255093441, "device_compute_slowdown": 0.9119109818181819, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-wvwj9evp/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "310a51aaac34ebd04d17becc4645f77793138816428bb43603834b11ab1687fb", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.028667, "capture_ms": 0.079083, "compile_ms": 57.141292, "allocation_upload_ms": 7.5695, "cold_call_ms": 11.122917, "warmup_ms": 200.765333, "download_ms": 3.111333, "repetitions": 320, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 93.789975, 89.51484375, 96.25625, 89.22513125, 96.805859375, 88.404296875, 96.92200625, 95.830078125, 89.00546875 ], "latency_us": [ 368.833, 366.584, 328.875, 2279.042, 763, 544.5, 265.542, 290.709, 360.667 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 72, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4500834, "compute_span_ns": 4500834, "calibration_cpu_ns": 5482125, "calibration_gpu_ticks": 5482125, "compute_passes": 1, "command_buffer_ns": 4500916.63515, "command_buffers": 2 }, { "compute_ns": 4505209, "compute_span_ns": 4505209, "calibration_cpu_ns": 5698416, "calibration_gpu_ticks": 5698416, "compute_passes": 1, "command_buffer_ns": 4505291.69757, "command_buffers": 2 }, { "compute_ns": 4475000, "compute_span_ns": 4475000, "calibration_cpu_ns": 5454666, "calibration_gpu_ticks": 5454666, "compute_passes": 1, "command_buffer_ns": 4475083.31854, "command_buffers": 2 }, { "compute_ns": 4597167, "compute_span_ns": 4597167, "calibration_cpu_ns": 5758625, "calibration_gpu_ticks": 5758625, "compute_passes": 1, "command_buffer_ns": 4597208.34624, "command_buffers": 2 }, { "compute_ns": 4303416, "compute_span_ns": 4303416, "calibration_cpu_ns": 5667833, "calibration_gpu_ticks": 5667833, "compute_passes": 1, "command_buffer_ns": 4303416.7029, "command_buffers": 2 }, { "compute_ns": 4451958, "compute_span_ns": 4451958, "calibration_cpu_ns": 5688709, "calibration_gpu_ticks": 5688709, "compute_passes": 1, "command_buffer_ns": 4452000.02287, "command_buffers": 2 }, { "compute_ns": 4430000, "compute_span_ns": 4430000, "calibration_cpu_ns": 5655834, "calibration_gpu_ticks": 5655834, "compute_passes": 1, "command_buffer_ns": 4430083.32513, "command_buffers": 2 }, { "compute_ns": 6780000, "compute_span_ns": 6780000, "calibration_cpu_ns": 7648916, "calibration_gpu_ticks": 7648916, "compute_passes": 1, "command_buffer_ns": 6780124.96792, "command_buffers": 2 }, { "compute_ns": 6595041, "compute_span_ns": 6595041, "calibration_cpu_ns": 7313083, "calibration_gpu_ticks": 7313083, "compute_passes": 1, "command_buffer_ns": 6595041.71927, "command_buffers": 2 } ], "latency": [ { "compute_ns": 106250, "compute_span_ns": 106250, "calibration_cpu_ns": 845916, "calibration_gpu_ticks": 845916, "compute_passes": 1, "command_buffer_ns": 106291.61261, "command_buffers": 2 }, { "compute_ns": 66500, "compute_span_ns": 66500, "calibration_cpu_ns": 554875, "calibration_gpu_ticks": 554875, "compute_passes": 1, "command_buffer_ns": 66541.6591801, "command_buffers": 2 }, { "compute_ns": 104042, "compute_span_ns": 104042, "calibration_cpu_ns": 803667, "calibration_gpu_ticks": 803667, "compute_passes": 1, "command_buffer_ns": 104041.711893, "command_buffers": 2 }, { "compute_ns": 64958, "compute_span_ns": 64958, "calibration_cpu_ns": 504250, "calibration_gpu_ticks": 504250, "compute_passes": 1, "command_buffer_ns": 65041.6477583, "command_buffers": 2 }, { "compute_ns": 65000, "compute_span_ns": 65000, "calibration_cpu_ns": 478291, "calibration_gpu_ticks": 478291, "compute_passes": 1, "command_buffer_ns": 65125.0011288, "command_buffers": 2 }, { "compute_ns": 236083, "compute_span_ns": 236083, "calibration_cpu_ns": 1832833, "calibration_gpu_ticks": 1832833, "compute_passes": 1, "command_buffer_ns": 236166.699324, "command_buffers": 2 }, { "compute_ns": 65708, "compute_span_ns": 65708, "calibration_cpu_ns": 726417, "calibration_gpu_ticks": 726417, "compute_passes": 1, "command_buffer_ns": 65791.5952615, "command_buffers": 2 }, { "compute_ns": 108834, "compute_span_ns": 108834, "calibration_cpu_ns": 839500, "calibration_gpu_ticks": 839500, "compute_passes": 1, "command_buffer_ns": 108958.28018, "command_buffers": 2 }, { "compute_ns": 109583, "compute_span_ns": 109583, "calibration_cpu_ns": 1095000, "calibration_gpu_ticks": 1095000, "compute_passes": 1, "command_buffer_ns": 109583.314043, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4401958.3147, "command_buffers": 2 }, { "command_buffer_ns": 4389624.98447, "command_buffers": 2 }, { "command_buffer_ns": 4445500.03158, "command_buffers": 2 }, { "command_buffer_ns": 4423500.03868, "command_buffers": 2 }, { "command_buffer_ns": 4541916.5981, "command_buffers": 2 }, { "command_buffer_ns": 4434749.94972, "command_buffers": 2 }, { "command_buffer_ns": 4481416.66129, "command_buffers": 2 }, { "command_buffer_ns": 6788708.38601, "command_buffers": 2 }, { "command_buffer_ns": 6754583.32989, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 68208.3773427, "command_buffers": 2 }, { "command_buffer_ns": 65750.0349917, "command_buffers": 2 }, { "command_buffer_ns": 106458.261143, "command_buffers": 2 }, { "command_buffer_ns": 108291.627839, "command_buffers": 2 }, { "command_buffer_ns": 64750.0273772, "command_buffers": 2 }, { "command_buffer_ns": 90499.9906197, "command_buffers": 2 }, { "command_buffer_ns": 65083.3826512, "command_buffers": 2 }, { "command_buffer_ns": 103875.005152, "command_buffers": 2 }, { "command_buffer_ns": 65041.705966, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 68.7805986671875, 68.58789038234374, 69.4609379934375, 69.117188104375, 70.9674468453125, 69.292967964375, 70.02213533265625, 106.07356853140624, 105.54036452953125 ], "command_buffer_throughput_us_p50": 69.4609379934375, "command_buffer_throughput_us_p90": 105.64700532990625, "command_buffer_latency_us": [ 68.2083773427, 65.7500349917, 106.458261143, 108.291627839, 64.7500273772, 90.4999906197, 65.0833826512, 103.875005152, 65.041705966 ], "command_buffer_latency_us_p50": 68.2083773427, "command_buffer_latency_us_p90": 106.8249344822 }, "compute_throughput_us": [ 70.32553125, 70.393890625, 69.921875, 71.830734375, 67.240875, 69.56184375, 69.21875, 105.9375, 103.047515625 ], "compute_throughput_us_p50": 70.32553125, "compute_throughput_us_p90": 103.6255125, "command_buffer_throughput_us": [ 70.32682242421876, 70.39518277453125, 69.92317685218751, 71.83138041, 67.2408859828125, 69.56250035734375, 69.22005195515625, 105.93945262375, 103.04752686359376 ], "command_buffer_throughput_us_p50": 70.32682242421876, "command_buffer_throughput_us_p90": 103.625912015625, "compute_latency_us": [ 106.25, 66.5, 104.042, 64.958, 65.0, 236.083, 65.708, 108.834, 109.583 ], "compute_latency_us_p50": 104.042, "compute_latency_us_p90": 134.883, "command_buffer_latency_us": [ 106.29161261, 66.5416591801, 104.041711893, 65.0416477583, 65.12500112880001, 236.16669932399998, 65.79159526149999, 108.95828018, 109.583314043 ], "command_buffer_latency_us_p50": 104.041711893, "command_buffer_latency_us_p90": 134.8999910992, "counter_control_throughput_ratio": 1.003889751022172, "counter_control_latency_ratio": 1.0120399052030913 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 93.789975, "throughput_us_p90": 96.82908875, "latency_us_p50": 366.584, "latency_us_p90": 1066.2084000000002 }, "torch": { "cold_call_ms": 8.187709, "warmup_ms": 201.766708, "repetitions": 383, "throughput_us": [ 106.63261618798956, 106.73128720626632, 105.0048955613577, 104.13142036553525, 98.8543315926893, 105.02959007832898, 104.95245953002612, 103.08159268929504, 97.96757963446476 ], "latency_us": [ 429.417, 312.125, 306.25, 299.75, 2353.708, 644.375, 458.5, 479.459, 313.583 ], "throughput_us_p50": 104.95245953002612, "throughput_us_p90": 106.6523503916449, "latency_us_p50": 429.417, "latency_us_p90": 986.2416000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 6821000.017225742, "command_buffers": 1 }, { "command_buffer_ns": 6666958.332061768, "command_buffers": 1 }, { "command_buffer_ns": 6735333.299729973, "command_buffers": 1 }, { "command_buffer_ns": 4659208.294469863, "command_buffers": 1 }, { "command_buffer_ns": 4663250.001613051, "command_buffers": 1 }, { "command_buffer_ns": 4585999.995470047, "command_buffers": 1 }, { "command_buffer_ns": 4466624.988708645, "command_buffers": 1 }, { "command_buffer_ns": 4559374.996460974, "command_buffers": 1 }, { "command_buffer_ns": 4753666.697070003, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 289166.6372306645, "command_buffers": 1 }, { "command_buffer_ns": 70124.9809935689, "command_buffers": 1 }, { "command_buffer_ns": 298666.6513606906, "command_buffers": 1 }, { "command_buffer_ns": 71166.66529327631, "command_buffers": 1 }, { "command_buffer_ns": 143791.66532307863, "command_buffers": 1 }, { "command_buffer_ns": 70374.98289719224, "command_buffers": 1 }, { "command_buffer_ns": 195749.97713789344, "command_buffers": 1 }, { "command_buffer_ns": 306125.0317841768, "command_buffers": 1 }, { "command_buffer_ns": 89208.30441638827, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 106.57812526915222, 104.17122393846512, 105.23958280828083, 72.80012960109161, 72.86328127520392, 71.65624992921948, 69.79101544857258, 71.24023431970272, 74.27604214171879 ], "command_buffer_throughput_us_p50": 72.86328127520392, "command_buffer_throughput_us_p90": 105.50729130045511, "command_buffer_latency_us": [ 289.1666372306645, 70.1249809935689, 298.6666513606906, 71.16666529327631, 143.79166532307863, 70.37498289719224, 195.74997713789344, 306.1250317841768, 89.20830441638827 ], "command_buffer_latency_us_p50": 143.79166532307863, "command_buffer_latency_us_p90": 300.15832744538784 }, "throughput": [ { "compute_ns": 6933333.0, "compute_span_ns": 6933333.0, "command_buffer_ns": 6933291.6755229235, "calibration_cpu_ns": 7918208.0, "calibration_gpu_ticks": 7918208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7005333.0, "compute_span_ns": 7005333.0, "command_buffer_ns": 7005291.699897498, "calibration_cpu_ns": 7962084.0, "calibration_gpu_ticks": 7962084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4343500.0, "compute_span_ns": 4343500.0, "command_buffer_ns": 4343500.011600554, "calibration_cpu_ns": 5412166.0, "calibration_gpu_ticks": 5412166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4616458.0, "compute_span_ns": 4616458.0, "command_buffer_ns": 4616458.318196237, "calibration_cpu_ns": 5876125.0, "calibration_gpu_ticks": 5876125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4521041.0, "compute_span_ns": 4521041.0, "command_buffer_ns": 4521000.024396926, "calibration_cpu_ns": 5888125.0, "calibration_gpu_ticks": 5888125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4687250.0, "compute_span_ns": 4687250.0, "command_buffer_ns": 4687249.951530248, "calibration_cpu_ns": 5991875.0, "calibration_gpu_ticks": 5991875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4561416.0, "compute_span_ns": 4561416.0, "command_buffer_ns": 4561375.011689961, "calibration_cpu_ns": 6109458.0, "calibration_gpu_ticks": 6109458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4539750.0, "compute_span_ns": 4539750.0, "command_buffer_ns": 4539749.992545694, "calibration_cpu_ns": 6035875.0, "calibration_gpu_ticks": 6035875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4471791.0, "compute_span_ns": 4471791.0, "command_buffer_ns": 4471749.998629093, "calibration_cpu_ns": 5898709.0, "calibration_gpu_ticks": 5898709.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 91250.0, "compute_span_ns": 91250.0, "command_buffer_ns": 91249.99633058906, "calibration_cpu_ns": 866791.0, "calibration_gpu_ticks": 866791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71667.0, "compute_span_ns": 71667.0, "command_buffer_ns": 71666.669100523, "calibration_cpu_ns": 549334.0, "calibration_gpu_ticks": 549334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88791.0, "compute_span_ns": 88791.0, "command_buffer_ns": 88749.97729435563, "calibration_cpu_ns": 666458.0, "calibration_gpu_ticks": 666458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72083.0, "compute_span_ns": 72083.0, "command_buffer_ns": 72041.70105978847, "calibration_cpu_ns": 496666.0, "calibration_gpu_ticks": 496666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 79459.0, "compute_span_ns": 79459.0, "command_buffer_ns": 79499.96506795287, "calibration_cpu_ns": 830959.0, "calibration_gpu_ticks": 830959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70334.0, "compute_span_ns": 70334.0, "command_buffer_ns": 70374.98289719224, "calibration_cpu_ns": 496292.0, "calibration_gpu_ticks": 496292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 73750.0, "compute_span_ns": 73750.0, "command_buffer_ns": 73749.9794922769, "calibration_cpu_ns": 627417.0, "calibration_gpu_ticks": 627417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71792.0, "compute_span_ns": 71792.0, "command_buffer_ns": 71833.31763371825, "calibration_cpu_ns": 642500.0, "calibration_gpu_ticks": 642500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70333.0, "compute_span_ns": 70333.0, "command_buffer_ns": 70291.68773442507, "calibration_cpu_ns": 558000.0, "calibration_gpu_ticks": 558000.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 108.333328125, 109.458328125, 67.8671875, 72.13215625, 70.641265625, 73.23828125, 71.272125, 70.93359375, 69.871734375 ], "compute_throughput_us_p50": 71.272125, "compute_throughput_us_p90": 108.55832812499999, "command_buffer_throughput_us": [ 108.33268243004568, 109.4576828108984, 67.86718768125866, 72.1321612218162, 70.64062538120197, 73.23828049266012, 71.27148455765564, 70.93359363352647, 69.87109372857958 ], "command_buffer_throughput_us_p50": 71.27148455765564, "command_buffer_throughput_us_p90": 108.55768250621622, "compute_latency_us": [ 91.25, 71.667, 88.791, 72.083, 79.459, 70.334, 73.75, 71.792, 70.333 ], "compute_latency_us_p50": 72.083, "compute_latency_us_p90": 89.2828, "command_buffer_latency_us": [ 91.24999633058906, 71.666669100523, 88.74997729435563, 72.04170105978847, 79.49996506795287, 70.37498289719224, 73.7499794922769, 71.83331763371825, 70.29168773442507 ], "command_buffer_latency_us_p50": 72.04170105978847, "command_buffer_latency_us_p90": 89.24998110160232, "counter_control_throughput_ratio": 0.995695681111882, "counter_control_latency_ratio": 0.5528829844854234 }, "download_ms": 1.027042, "allocation_upload_ms": 19.1575, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.8936424684089217, "device_compute_slowdown": 0.9867185979090702, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.355708, "warmup_ms": 200.969333, "repetitions": 341, "throughput_us": [ 101.82673607038123, 106.0294486803519, 102.44085923753666, 102.28274486803521, 102.57685630498533, 108.95246920821114, 102.06365982404692, 101.38526392961877, 102.02981231671555 ], "latency_us": [ 433.542, 339.75, 339.167, 2345.2920000000004, 678.375, 451.042, 427.416, 319.75, 317.125 ], "throughput_us_p50": 102.28274486803521, "throughput_us_p90": 106.61405278592375, "latency_us_p50": 427.416, "latency_us_p90": 1011.7584000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 6890249.962452799, "command_buffers": 1 }, { "command_buffer_ns": 6845000.0253506005, "command_buffers": 1 }, { "command_buffer_ns": 6682541.687041521, "command_buffers": 1 }, { "command_buffer_ns": 4464041.6745096445, "command_buffers": 1 }, { "command_buffer_ns": 4826583.317480981, "command_buffers": 1 }, { "command_buffer_ns": 4576041.654217988, "command_buffers": 1 }, { "command_buffer_ns": 4641416.657250375, "command_buffers": 1 }, { "command_buffer_ns": 4460000.025574118, "command_buffers": 1 }, { "command_buffer_ns": 4650666.669476777, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 310958.3049081266, "command_buffers": 1 }, { "command_buffer_ns": 70500.01295283437, "command_buffers": 1 }, { "command_buffer_ns": 143708.31195265055, "command_buffers": 1 }, { "command_buffer_ns": 70500.01295283437, "command_buffers": 1 }, { "command_buffer_ns": 192624.9824464321, "command_buffers": 1 }, { "command_buffer_ns": 76541.67711734772, "command_buffers": 1 }, { "command_buffer_ns": 337541.62723198533, "command_buffers": 1 }, { "command_buffer_ns": 70291.68773442507, "command_buffers": 1 }, { "command_buffer_ns": 296291.6623800993, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 107.66015566332499, 106.95312539610313, 104.41471386002377, 69.7506511642132, 75.41536433564033, 71.50065084715607, 72.5221352695371, 69.68750039959559, 72.66666671057465 ], "command_buffer_throughput_us_p50": 72.66666671057465, "command_buffer_throughput_us_p90": 107.0945314495475, "command_buffer_latency_us": [ 310.9583049081266, 70.50001295283437, 143.70831195265055, 70.50001295283437, 192.6249824464321, 76.54167711734772, 337.54162723198533, 70.29168773442507, 296.2916623800993 ], "command_buffer_latency_us_p50": 143.70831195265055, "command_buffer_latency_us_p90": 316.27496937289834 }, "throughput": [ { "compute_ns": 6881333.0, "compute_span_ns": 6881333.0, "command_buffer_ns": 6881291.687022895, "calibration_cpu_ns": 7996000.0, "calibration_gpu_ticks": 7996000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 6934458.0, "compute_span_ns": 6934458.0, "command_buffer_ns": 6934458.331670612, "calibration_cpu_ns": 7935625.0, "calibration_gpu_ticks": 7935625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 6199583.0, "compute_span_ns": 6199583.0, "command_buffer_ns": 6199583.294801414, "calibration_cpu_ns": 7573875.0, "calibration_gpu_ticks": 7573875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5341583.0, "compute_span_ns": 5341583.0, "command_buffer_ns": 5341583.339031786, "calibration_cpu_ns": 7033125.0, "calibration_gpu_ticks": 7033125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4737667.0, "compute_span_ns": 4737667.0, "command_buffer_ns": 4737666.69165343, "calibration_cpu_ns": 6176375.0, "calibration_gpu_ticks": 6176375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4722500.0, "compute_span_ns": 4722500.0, "command_buffer_ns": 4722499.987110496, "calibration_cpu_ns": 6077000.0, "calibration_gpu_ticks": 6077000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4552333.0, "compute_span_ns": 4552333.0, "command_buffer_ns": 4552291.647996753, "calibration_cpu_ns": 5980083.0, "calibration_gpu_ticks": 5980083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4585375.0, "compute_span_ns": 4585375.0, "command_buffer_ns": 4585375.019814819, "calibration_cpu_ns": 5953125.0, "calibration_gpu_ticks": 5953125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4732042.0, "compute_span_ns": 4732042.0, "command_buffer_ns": 4732083.35461095, "calibration_cpu_ns": 6114750.0, "calibration_gpu_ticks": 6114750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 90833.0, "compute_span_ns": 90833.0, "command_buffer_ns": 90791.66920855641, "calibration_cpu_ns": 793125.0, "calibration_gpu_ticks": 793125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70875.0, "compute_span_ns": 70875.0, "command_buffer_ns": 70874.98670443892, "calibration_cpu_ns": 539875.0, "calibration_gpu_ticks": 539875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 90666.0, "compute_span_ns": 90666.0, "command_buffer_ns": 90625.02067536116, "calibration_cpu_ns": 1026125.0, "calibration_gpu_ticks": 1026125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 76125.0, "compute_span_ns": 76125.0, "command_buffer_ns": 76124.9684728682, "calibration_cpu_ns": 533792.0, "calibration_gpu_ticks": 533792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 85292.0, "compute_span_ns": 85292.0, "command_buffer_ns": 85291.68553650379, "calibration_cpu_ns": 814375.0, "calibration_gpu_ticks": 814375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70917.0, "compute_span_ns": 70917.0, "command_buffer_ns": 70958.34007486701, "calibration_cpu_ns": 495375.0, "calibration_gpu_ticks": 495375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 100625.0, "compute_span_ns": 100625.0, "command_buffer_ns": 100624.98040497303, "calibration_cpu_ns": 942792.0, "calibration_gpu_ticks": 942792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71042.0, "compute_span_ns": 71042.0, "command_buffer_ns": 71083.31192284822, "calibration_cpu_ns": 548625.0, "calibration_gpu_ticks": 548625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 89875.0, "compute_span_ns": 89875.0, "command_buffer_ns": 89875.01496449113, "calibration_cpu_ns": 884916.0, "calibration_gpu_ticks": 884916.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 107.520828125, 108.35090625, 96.868484375, 83.462234375, 74.026046875, 73.7890625, 71.130203125, 71.646484375, 73.93815625 ], "compute_throughput_us_p50": 74.026046875, "compute_throughput_us_p90": 107.68684375, "command_buffer_throughput_us": [ 107.52018260973273, 108.35091143235331, 96.8684889812721, 83.46223967237165, 74.02604205708485, 73.7890622986015, 71.12955699994927, 71.64648468460655, 73.93880241579609 ], "command_buffer_throughput_us_p50": 74.02604205708485, "command_buffer_throughput_us_p90": 107.68632837425685, "compute_latency_us": [ 90.833, 70.875, 90.666, 76.125, 85.292, 70.917, 100.625, 71.042, 89.875 ], "compute_latency_us_p50": 85.292, "compute_latency_us_p90": 92.7914, "command_buffer_latency_us": [ 90.79166920855641, 70.87498670443892, 90.62502067536116, 76.1249684728682, 85.29168553650379, 70.95834007486701, 100.62498040497303, 71.08331192284822, 89.87501496449113 ], "command_buffer_latency_us_p50": 85.29168553650379, "command_buffer_latency_us_p90": 92.75833144783974, "counter_control_throughput_ratio": 1.0130691462364794, "counter_control_latency_ratio": 0.6306178080027867 }, "download_ms": 0.626875, "allocation_upload_ms": 1.227167, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-uxrkr7y0/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "26111b6b600e52c50494ad3f783ef6fc3b02252d329d47ea39eb66749773d18e", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.078042, "capture_ms": 0.071, "compile_ms": 55.432375, "allocation_upload_ms": 5.77925, "cold_call_ms": 6.844584, "warmup_ms": 200.476167, "download_ms": 2.974417, "repetitions": 381, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 74.0955826772, 81.4888451444, 73.4520997375, 80.687335958, 73.6167979003, 81.3713910761, 75.1672152231, 77.7517506562, 73.8311469816 ], "latency_us": [ 1658.375, 2098.209, 325.708, 290.125, 281.083, 299.375, 269.292, 282.75, 299.875 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 1280, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3654791, "compute_span_ns": 3654791, "calibration_cpu_ns": 4909875, "calibration_gpu_ticks": 4909875, "compute_passes": 1, "command_buffer_ns": 3654874.91945, "command_buffers": 2 }, { "compute_ns": 3782458, "compute_span_ns": 3782458, "calibration_cpu_ns": 4804834, "calibration_gpu_ticks": 4804834, "compute_passes": 1, "command_buffer_ns": 3782499.98903, "command_buffers": 2 }, { "compute_ns": 3648166, "compute_span_ns": 3648166, "calibration_cpu_ns": 4901166, "calibration_gpu_ticks": 4901166, "compute_passes": 1, "command_buffer_ns": 3648208.33784, "command_buffers": 2 }, { "compute_ns": 3716916, "compute_span_ns": 3716916, "calibration_cpu_ns": 4801709, "calibration_gpu_ticks": 4801709, "compute_passes": 1, "command_buffer_ns": 3717000.07236, "command_buffers": 2 }, { "compute_ns": 3906500, "compute_span_ns": 3906500, "calibration_cpu_ns": 4864292, "calibration_gpu_ticks": 4864292, "compute_passes": 1, "command_buffer_ns": 3906625.03196, "command_buffers": 2 }, { "compute_ns": 3837083, "compute_span_ns": 3837083, "calibration_cpu_ns": 4850834, "calibration_gpu_ticks": 4850834, "compute_passes": 1, "command_buffer_ns": 3837208.32178, "command_buffers": 2 }, { "compute_ns": 3664000, "compute_span_ns": 3664000, "calibration_cpu_ns": 4864083, "calibration_gpu_ticks": 4864083, "compute_passes": 1, "command_buffer_ns": 3664125.04809, "command_buffers": 2 }, { "compute_ns": 3767375, "compute_span_ns": 3767375, "calibration_cpu_ns": 4962416, "calibration_gpu_ticks": 4962416, "compute_passes": 1, "command_buffer_ns": 3767458.25633, "command_buffers": 2 }, { "compute_ns": 3598292, "compute_span_ns": 3598292, "calibration_cpu_ns": 4801083, "calibration_gpu_ticks": 4801083, "compute_passes": 1, "command_buffer_ns": 3598458.30826, "command_buffers": 2 } ], "latency": [ { "compute_ns": 96166, "compute_span_ns": 96166, "calibration_cpu_ns": 790625, "calibration_gpu_ticks": 790625, "compute_passes": 1, "command_buffer_ns": 96125.0043474, "command_buffers": 2 }, { "compute_ns": 54250, "compute_span_ns": 54250, "calibration_cpu_ns": 566500, "calibration_gpu_ticks": 566500, "compute_passes": 1, "command_buffer_ns": 54333.3007954, "command_buffers": 2 }, { "compute_ns": 64334, "compute_span_ns": 64334, "calibration_cpu_ns": 490834, "calibration_gpu_ticks": 490834, "compute_passes": 1, "command_buffer_ns": 64374.9954179, "command_buffers": 2 }, { "compute_ns": 215875, "compute_span_ns": 215875, "calibration_cpu_ns": 1501250, "calibration_gpu_ticks": 1501250, "compute_passes": 1, "command_buffer_ns": 215916.661546, "command_buffers": 2 }, { "compute_ns": 271584, "compute_span_ns": 271584, "calibration_cpu_ns": 1624666, "calibration_gpu_ticks": 1624666, "compute_passes": 1, "command_buffer_ns": 271875.003818, "command_buffers": 2 }, { "compute_ns": 88541, "compute_span_ns": 88541, "calibration_cpu_ns": 738375, "calibration_gpu_ticks": 738375, "compute_passes": 1, "command_buffer_ns": 88541.6520759, "command_buffers": 2 }, { "compute_ns": 233042, "compute_span_ns": 233042, "calibration_cpu_ns": 1723125, "calibration_gpu_ticks": 1723125, "compute_passes": 1, "command_buffer_ns": 233124.999795, "command_buffers": 2 }, { "compute_ns": 96291, "compute_span_ns": 96291, "calibration_cpu_ns": 880750, "calibration_gpu_ticks": 880750, "compute_passes": 1, "command_buffer_ns": 96333.3295658, "command_buffers": 2 }, { "compute_ns": 195042, "compute_span_ns": 195042, "calibration_cpu_ns": 1668875, "calibration_gpu_ticks": 1668875, "compute_passes": 1, "command_buffer_ns": 195249.973331, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3940916.62019, "command_buffers": 2 }, { "command_buffer_ns": 3860249.99898, "command_buffers": 2 }, { "command_buffer_ns": 3739624.9827, "command_buffers": 2 }, { "command_buffer_ns": 3689083.32894, "command_buffers": 2 }, { "command_buffer_ns": 3737125.02187, "command_buffers": 2 }, { "command_buffer_ns": 3667416.69131, "command_buffers": 2 }, { "command_buffer_ns": 3842208.30165, "command_buffers": 2 }, { "command_buffer_ns": 3744375.01887, "command_buffers": 2 }, { "command_buffer_ns": 3719166.67791, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 286291.586235, "command_buffers": 2 }, { "command_buffer_ns": 254583.253991, "command_buffers": 2 }, { "command_buffer_ns": 99000.0553429, "command_buffers": 2 }, { "command_buffer_ns": 95583.3820626, "command_buffers": 2 }, { "command_buffer_ns": 53541.676607, "command_buffers": 2 }, { "command_buffer_ns": 53583.3532922, "command_buffers": 2 }, { "command_buffer_ns": 53291.6747034, "command_buffers": 2 }, { "command_buffer_ns": 54291.6823179, "command_buffers": 2 }, { "command_buffer_ns": 54000.0619367, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 61.576822190468754, 60.316406234062505, 58.4316403546875, 57.6419270146875, 58.392578466718746, 57.303385801718754, 60.034504713281244, 58.505859669843744, 58.11197934234375 ], "command_buffer_throughput_us_p50": 58.4316403546875, "command_buffer_throughput_us_p90": 60.568489425343756, "command_buffer_latency_us": [ 286.291586235, 254.583253991, 99.0000553429, 95.5833820626, 53.541676607, 53.5833532922, 53.2916747034, 54.2916823179, 54.00006193670001 ], "command_buffer_latency_us_p50": 54.2916823179, "command_buffer_latency_us_p90": 260.9249204398 }, "compute_throughput_us": [ 57.106109375, 59.10090625, 57.00259375, 58.0768125, 61.0390625, 59.954421875, 57.25, 58.865234375, 56.2233125 ], "compute_throughput_us_p50": 58.0768125, "compute_throughput_us_p90": 60.171350000000004, "command_buffer_throughput_us": [ 57.10742061640625, 59.101562328593744, 57.003255278750004, 58.078126130625, 61.041016124375, 59.9563800278125, 57.251953876406255, 58.86653525515624, 56.225911066562496 ], "command_buffer_throughput_us_p50": 58.078126130625, "command_buffer_throughput_us_p90": 60.173307247125, "compute_latency_us": [ 96.166, 54.25, 64.334, 215.875, 271.584, 88.541, 233.042, 96.291, 195.042 ], "compute_latency_us_p50": 96.291, "compute_latency_us_p90": 240.7504, "command_buffer_latency_us": [ 96.1250043474, 54.3333007954, 64.3749954179, 215.916661546, 271.87500381800004, 88.5416520759, 233.124999795, 96.3333295658, 195.24997333099998 ], "command_buffer_latency_us_p50": 96.3333295658, "command_buffer_latency_us_p90": 240.8750005996, "counter_control_throughput_ratio": 0.9798588148512287, "counter_control_latency_ratio": 1.7743662648309362 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 75.1672152231, "throughput_us_p90": 81.39488188976, "latency_us_p50": 299.375, "latency_us_p90": 1746.3418000000001 }, "slowdown": 0.7348963436607069, "device_compute_slowdown": 0.7845456424016293, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-2xjrlcfu/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "90a0b6dbe0fbfeffc09b3a8db9fc5856f9e2631dc2ce0e6d9ee991918165a6cf", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.942208, "capture_ms": 0.084125, "compile_ms": 65.268083, "allocation_upload_ms": 3.271375, "cold_call_ms": 2.172375, "warmup_ms": 200.039625, "download_ms": 0.301209, "repetitions": 3532, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.61332106455, 5.88396545866, 6.48290628539, 7.10185447339, 6.63008437146, 6.27242582106, 5.9587698188, 6.62937684032, 6.48703510759 ], "latency_us": [ 309.292, 454.458, 209.25, 200.542, 192.167, 200.334, 288.292, 205.125, 201.291 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 313917, "compute_span_ns": 313917, "calibration_cpu_ns": 785750, "calibration_gpu_ticks": 785750, "compute_passes": 1, "command_buffer_ns": 313958.327752, "command_buffers": 2 }, { "compute_ns": 683166, "compute_span_ns": 683166, "calibration_cpu_ns": 2196584, "calibration_gpu_ticks": 2196584, "compute_passes": 1, "command_buffer_ns": 683249.963913, "command_buffers": 2 }, { "compute_ns": 726750, "compute_span_ns": 726750, "calibration_cpu_ns": 2359958, "calibration_gpu_ticks": 2359958, "compute_passes": 1, "command_buffer_ns": 726874.975953, "command_buffers": 2 }, { "compute_ns": 295375, "compute_span_ns": 295375, "calibration_cpu_ns": 861708, "calibration_gpu_ticks": 861708, "compute_passes": 1, "command_buffer_ns": 295416.626614, "command_buffers": 2 }, { "compute_ns": 292333, "compute_span_ns": 292333, "calibration_cpu_ns": 733917, "calibration_gpu_ticks": 733917, "compute_passes": 1, "command_buffer_ns": 292333.366815, "command_buffers": 2 }, { "compute_ns": 265375, "compute_span_ns": 265375, "calibration_cpu_ns": 1006500, "calibration_gpu_ticks": 1006500, "compute_passes": 1, "command_buffer_ns": 265499.926172, "command_buffers": 2 }, { "compute_ns": 279708, "compute_span_ns": 279708, "calibration_cpu_ns": 715916, "calibration_gpu_ticks": 715916, "compute_passes": 1, "command_buffer_ns": 279708.357994, "command_buffers": 2 }, { "compute_ns": 711959, "compute_span_ns": 711959, "calibration_cpu_ns": 2252083, "calibration_gpu_ticks": 2252083, "compute_passes": 1, "command_buffer_ns": 712041.684892, "command_buffers": 2 }, { "compute_ns": 776958, "compute_span_ns": 776958, "calibration_cpu_ns": 2311125, "calibration_gpu_ticks": 2311125, "compute_passes": 1, "command_buffer_ns": 777041.597757, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6750, "compute_span_ns": 6750, "calibration_cpu_ns": 494459, "calibration_gpu_ticks": 494459, "compute_passes": 1, "command_buffer_ns": 6874.96503815, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 402375, "calibration_gpu_ticks": 402375, "compute_passes": 1, "command_buffer_ns": 7333.29216018, "command_buffers": 2 }, { "compute_ns": 6791, "compute_span_ns": 6791, "calibration_cpu_ns": 398416, "calibration_gpu_ticks": 398416, "compute_passes": 1, "command_buffer_ns": 6791.66987538, "command_buffers": 2 }, { "compute_ns": 7208, "compute_span_ns": 7208, "calibration_cpu_ns": 699750, "calibration_gpu_ticks": 699750, "compute_passes": 1, "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "compute_ns": 6917, "compute_span_ns": 6917, "calibration_cpu_ns": 376000, "calibration_gpu_ticks": 376000, "compute_passes": 1, "command_buffer_ns": 7041.72998667, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 476583, "calibration_gpu_ticks": 476583, "compute_passes": 1, "command_buffer_ns": 7249.99699742, "command_buffers": 2 }, { "compute_ns": 7292, "compute_span_ns": 7292, "calibration_cpu_ns": 821500, "calibration_gpu_ticks": 821500, "compute_passes": 1, "command_buffer_ns": 7333.29216018, "command_buffers": 2 }, { "compute_ns": 6750, "compute_span_ns": 6750, "calibration_cpu_ns": 360291, "calibration_gpu_ticks": 360291, "compute_passes": 1, "command_buffer_ns": 6833.28835294, "command_buffers": 2 }, { "compute_ns": 8625, "compute_span_ns": 8625, "calibration_cpu_ns": 758500, "calibration_gpu_ticks": 758500, "compute_passes": 1, "command_buffer_ns": 8666.71325639, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 682083.365973, "command_buffers": 2 }, { "command_buffer_ns": 295875.011943, "command_buffers": 2 }, { "command_buffer_ns": 289416.639134, "command_buffers": 2 }, { "command_buffer_ns": 266916.642431, "command_buffers": 2 }, { "command_buffer_ns": 549916.643649, "command_buffers": 2 }, { "command_buffer_ns": 658416.654915, "command_buffers": 2 }, { "command_buffer_ns": 306500.005536, "command_buffers": 2 }, { "command_buffer_ns": 293333.316222, "command_buffers": 2 }, { "command_buffer_ns": 266333.285253, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6958.37661624, "command_buffers": 2 }, { "command_buffer_ns": 9916.72277451, "command_buffers": 2 }, { "command_buffer_ns": 7666.7056419, "command_buffers": 2 }, { "command_buffer_ns": 44958.316721, "command_buffers": 2 }, { "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "command_buffer_ns": 7249.99699742, "command_buffers": 2 }, { "command_buffer_ns": 76874.9741837, "command_buffers": 2 }, { "command_buffer_ns": 9458.33744481, "command_buffers": 2 }, { "command_buffer_ns": 42208.3539888, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 10.657552593328123, 4.623047061609375, 4.52213498646875, 4.170572537984375, 8.592447557015625, 10.287760233046875, 4.7890625865, 4.58333306596875, 4.161457582078125 ], "command_buffer_throughput_us_p50": 4.623047061609375, "command_buffer_throughput_us_p90": 10.361718705103124, "command_buffer_latency_us": [ 6.95837661624, 9.916722774510001, 7.6667056419, 44.958316721, 7.708324119449999, 7.24999699742, 76.8749741837, 9.45833744481, 42.2083539888 ], "command_buffer_latency_us_p50": 9.45833744481, "command_buffer_latency_us_p90": 51.34164821354001 }, "compute_throughput_us": [ 4.904953125, 10.67446875, 11.35546875, 4.615234375, 4.567703125, 4.146484375, 4.3704375, 11.124359375, 12.13996875 ], "compute_throughput_us_p50": 4.904953125, "compute_throughput_us_p90": 11.51236875, "command_buffer_throughput_us": [ 4.905598871125, 10.675780686140625, 11.357421499265625, 4.61588479084375, 4.5677088564843755, 4.1484363464375, 4.370443093656251, 11.1256513264375, 12.141274964953125 ], "command_buffer_throughput_us_p50": 4.905598871125, "command_buffer_throughput_us_p90": 11.514192192403124, "compute_latency_us": [ 6.75, 7.25, 6.791, 7.208, 6.917, 7.25, 7.292, 6.75, 8.625 ], "compute_latency_us_p50": 7.208, "compute_latency_us_p90": 7.5586, "command_buffer_latency_us": [ 6.87496503815, 7.33329216018, 6.79166987538, 7.1667018346499995, 7.041729986669999, 7.24999699742, 7.33329216018, 6.8332883529399995, 8.666713256389999 ], "command_buffer_latency_us_p50": 7.1667018346499995, "command_buffer_latency_us_p90": 7.599976379422, "counter_control_throughput_ratio": 1.1067748489694398, "counter_control_latency_ratio": 0.7394874624336111 }, "correctness": { "max_abs_error": 2.43199801630567e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 6.48703510759, "throughput_us_p90": 6.724438391846, "latency_us_p50": 205.125, "latency_us_p90": 338.3252 }, "torch": { "cold_call_ms": 11.251292, "warmup_ms": 200.164042, "repetitions": 1956, "throughput_us": [ 14.733895705521473, 14.84270654396728, 14.53007873210634, 16.527053680981595, 15.267574130879346, 14.012461656441717, 15.31639826175869, 13.991244887525562, 14.254367075664621 ], "latency_us": [ 1366.792, 389.625, 418.792, 514.75, 244.834, 218.95800000000003, 217.209, 217.917, 291.25 ], "throughput_us_p50": 14.733895705521473, "throughput_us_p90": 15.558529345603272, "latency_us_p50": 291.25, "latency_us_p90": 685.1584000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 646833.3303928375, "command_buffers": 1 }, { "command_buffer_ns": 474250.0022985041, "command_buffers": 1 }, { "command_buffer_ns": 559958.3382718265, "command_buffers": 1 }, { "command_buffer_ns": 471124.9493993819, "command_buffers": 1 }, { "command_buffer_ns": 650416.6522063315, "command_buffers": 1 }, { "command_buffer_ns": 472624.96082112193, "command_buffers": 1 }, { "command_buffer_ns": 661416.6777580976, "command_buffers": 1 }, { "command_buffer_ns": 469749.96803328395, "command_buffers": 1 }, { "command_buffer_ns": 609958.3697505295, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 78875.04762038589, "command_buffers": 1 }, { "command_buffer_ns": 146874.9833293259, "command_buffers": 1 }, { "command_buffer_ns": 10583.316907286644, "command_buffers": 1 }, { "command_buffer_ns": 116791.63435474038, "command_buffers": 1 }, { "command_buffer_ns": 11124.997399747372, "command_buffers": 1 }, { "command_buffer_ns": 12374.94871020317, "command_buffers": 1 }, { "command_buffer_ns": 10291.696526110172, "command_buffers": 1 }, { "command_buffer_ns": 9874.98788163066, "command_buffers": 1 }, { "command_buffer_ns": 10000.017937272787, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 10.106770787388086, 7.410156285914127, 8.74934903549729, 7.361327334365342, 10.16276019072393, 7.38476501283003, 10.334635589970276, 7.339843250520062, 9.530599527352024 ], "command_buffer_throughput_us_p50": 8.74934903549729, "command_buffer_throughput_us_p90": 10.197135270573199, "command_buffer_latency_us": [ 78.87504762038589, 146.87498332932591, 10.583316907286644, 116.79163435474038, 11.124997399747372, 12.37494871020317, 10.291696526110172, 9.874987881630659, 10.000017937272787 ], "command_buffer_latency_us_p50": 11.124997399747372, "command_buffer_latency_us_p90": 122.80830414965749 }, "throughput": [ { "compute_ns": 495875.0, "compute_span_ns": 495875.0, "command_buffer_ns": 495874.96323511004, "calibration_cpu_ns": 1356125.0, "calibration_gpu_ticks": 1356125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 687541.0, "compute_span_ns": 687541.0, "command_buffer_ns": 687499.9962747097, "calibration_cpu_ns": 2245833.0, "calibration_gpu_ticks": 2245833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485166.0, "compute_span_ns": 485166.0, "command_buffer_ns": 485124.99779462814, "calibration_cpu_ns": 1261000.0, "calibration_gpu_ticks": 1261000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 546834.0, "compute_span_ns": 546834.0, "command_buffer_ns": 546875.0023283064, "calibration_cpu_ns": 1941791.0, "calibration_gpu_ticks": 1941791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 470958.0, "compute_span_ns": 470958.0, "command_buffer_ns": 470916.6823886335, "calibration_cpu_ns": 1237583.0, "calibration_gpu_ticks": 1237583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 656125.0, "compute_span_ns": 656125.0, "command_buffer_ns": 656125.0193044543, "calibration_cpu_ns": 2157625.0, "calibration_gpu_ticks": 2157625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 468000.0, "compute_span_ns": 468000.0, "command_buffer_ns": 468000.01291558146, "calibration_cpu_ns": 1219125.0, "calibration_gpu_ticks": 1219125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 717625.0, "compute_span_ns": 717625.0, "command_buffer_ns": 717625.0219345093, "calibration_cpu_ns": 2272542.0, "calibration_gpu_ticks": 2272542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 481292.0, "compute_span_ns": 481292.0, "command_buffer_ns": 481333.3507627249, "calibration_cpu_ns": 1315750.0, "calibration_gpu_ticks": 1315750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 10958.0, "compute_span_ns": 10958.0, "command_buffer_ns": 10916.672181338072, "calibration_cpu_ns": 902125.0, "calibration_gpu_ticks": 902125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10292.0, "compute_span_ns": 10292.0, "command_buffer_ns": 10333.373211324215, "calibration_cpu_ns": 777208.0, "calibration_gpu_ticks": 777208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9625.0, "compute_span_ns": 9625.0, "command_buffer_ns": 9624.985978007317, "calibration_cpu_ns": 515042.0, "calibration_gpu_ticks": 515042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10125.0, "compute_span_ns": 10125.0, "command_buffer_ns": 10124.989785254002, "calibration_cpu_ns": 446625.0, "calibration_gpu_ticks": 446625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10167.0, "compute_span_ns": 10167.0, "command_buffer_ns": 10208.343155682087, "calibration_cpu_ns": 578000.0, "calibration_gpu_ticks": 578000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 31542.0, "compute_span_ns": 31542.0, "command_buffer_ns": 31541.625503450632, "calibration_cpu_ns": 1334334.0, "calibration_gpu_ticks": 1334334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 75250.0, "compute_span_ns": 75250.0, "command_buffer_ns": 75249.99091401696, "calibration_cpu_ns": 1594542.0, "calibration_gpu_ticks": 1594542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10792.0, "compute_span_ns": 10792.0, "command_buffer_ns": 10833.318810909986, "calibration_cpu_ns": 806500.0, "calibration_gpu_ticks": 806500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 115875.0, "compute_span_ns": 115875.0, "command_buffer_ns": 115874.9801106751, "calibration_cpu_ns": 1245166.0, "calibration_gpu_ticks": 1245166.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 7.748046875, 10.742828125, 7.58071875, 8.54428125, 7.35871875, 10.251953125, 7.3125, 11.212890625, 7.5201875 ], "compute_throughput_us_p50": 7.748046875, "compute_throughput_us_p90": 10.836840625, "command_buffer_throughput_us": [ 7.7480463005485944, 10.742187441792339, 7.580078090541065, 8.544921911379788, 7.358073162322398, 10.251953426632099, 7.31250020180596, 11.212890967726707, 7.520833605667576 ], "command_buffer_throughput_us_p50": 7.7480463005485944, "command_buffer_throughput_us_p90": 10.836328146979213, "compute_latency_us": [ 10.958, 10.292, 9.625, 10.125, 10.167, 31.542, 75.25, 10.792, 115.875 ], "compute_latency_us_p50": 10.792, "compute_latency_us_p90": 83.375, "command_buffer_latency_us": [ 10.916672181338072, 10.333373211324215, 9.624985978007317, 10.124989785254002, 10.208343155682087, 31.541625503450632, 75.24999091401696, 10.833318810909986, 115.8749801106751 ], "command_buffer_latency_us_p50": 10.833318810909986, "command_buffer_latency_us_p90": 83.37498875334859, "counter_control_throughput_ratio": 0.8663590925207881, "counter_control_latency_ratio": 0.9176040936345656 }, "download_ms": 0.767542, "allocation_upload_ms": 1.000416, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.4402796950136553, "device_compute_slowdown": 0.6330567179228636, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.330917, "warmup_ms": 200.2135, "repetitions": 1747, "throughput_us": [ 14.621708643388667, 14.54531597023469, 14.768603319977103, 15.282889524899828, 17.763189467658844, 15.212841442472811, 16.713199198626217, 14.901378362907842, 14.994132799084145 ], "latency_us": [ 348.375, 242.667, 258.125, 212.959, 256.0, 289.291, 225.667, 204.5, 287.25 ], "throughput_us_p50": 14.994132799084145, "throughput_us_p90": 16.923197252432743, "latency_us_p50": 256.0, "latency_us_p90": 301.1078, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 708750.041667372, "command_buffers": 1 }, { "command_buffer_ns": 477750.0289492309, "command_buffers": 1 }, { "command_buffer_ns": 643291.6852645576, "command_buffers": 1 }, { "command_buffer_ns": 468708.34194123745, "command_buffers": 1 }, { "command_buffer_ns": 572499.9937228858, "command_buffers": 1 }, { "command_buffer_ns": 767749.9670535326, "command_buffers": 1 }, { "command_buffer_ns": 495625.0195391476, "command_buffers": 1 }, { "command_buffer_ns": 610875.0239945948, "command_buffers": 1 }, { "command_buffer_ns": 508875.00401586294, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 17625.046893954277, "command_buffers": 1 }, { "command_buffer_ns": 10041.69462248683, "command_buffers": 1 }, { "command_buffer_ns": 79500.02327561378, "command_buffers": 1 }, { "command_buffer_ns": 10166.666470468044, "command_buffers": 1 }, { "command_buffer_ns": 13291.661161929369, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10000.017937272787, "command_buffers": 1 }, { "command_buffer_ns": 9499.955922365189, "command_buffers": 1 }, { "command_buffer_ns": 25208.34095776081, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 11.074219401052687, 7.464844202331733, 10.051432582258713, 7.323567842831835, 8.945312401920091, 11.996093235211447, 7.7441409302991815, 9.544922249915544, 7.9511719377478585 ], "command_buffer_throughput_us_p50": 8.945312401920091, "command_buffer_throughput_us_p90": 11.25859416788444, "command_buffer_latency_us": [ 17.625046893954277, 10.04169462248683, 79.50002327561378, 10.166666470468044, 13.291661161929369, 10.25001984089613, 10.000017937272787, 9.499955922365189, 25.20834095776081 ], "command_buffer_latency_us_p50": 10.25001984089613, "command_buffer_latency_us_p90": 36.06667742133141 }, "throughput": [ { "compute_ns": 494541.0, "compute_span_ns": 494541.0, "command_buffer_ns": 494499.9818690121, "calibration_cpu_ns": 1374250.0, "calibration_gpu_ticks": 1374250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 578708.0, "compute_span_ns": 578708.0, "command_buffer_ns": 578708.3064205945, "calibration_cpu_ns": 2381292.0, "calibration_gpu_ticks": 2381292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 487125.0, "compute_span_ns": 487125.0, "command_buffer_ns": 487125.0130236149, "calibration_cpu_ns": 1220833.0, "calibration_gpu_ticks": 1220833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 656667.0, "compute_span_ns": 656667.0, "command_buffer_ns": 656666.699796915, "calibration_cpu_ns": 2419917.0, "calibration_gpu_ticks": 2419917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 476167.0, "compute_span_ns": 476167.0, "command_buffer_ns": 476208.3408422768, "calibration_cpu_ns": 1172167.0, "calibration_gpu_ticks": 1172167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 524917.0, "compute_span_ns": 524917.0, "command_buffer_ns": 524958.3045952022, "calibration_cpu_ns": 1324375.0, "calibration_gpu_ticks": 1324375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 566250.0, "compute_span_ns": 566250.0, "command_buffer_ns": 566250.0043399632, "calibration_cpu_ns": 2382083.0, "calibration_gpu_ticks": 2382083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 493500.0, "compute_span_ns": 493500.0, "command_buffer_ns": 493499.97425451875, "calibration_cpu_ns": 1262458.0, "calibration_gpu_ticks": 1262458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 475208.0, "compute_span_ns": 475208.0, "command_buffer_ns": 475166.6565425694, "calibration_cpu_ns": 1148959.0, "calibration_gpu_ticks": 1148959.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 11125.0, "compute_span_ns": 11125.0, "command_buffer_ns": 11124.997399747372, "calibration_cpu_ns": 750167.0, "calibration_gpu_ticks": 750167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9875.0, "compute_span_ns": 9875.0, "command_buffer_ns": 9875.046089291573, "calibration_cpu_ns": 404917.0, "calibration_gpu_ticks": 404917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10833.0, "compute_span_ns": 10833.0, "command_buffer_ns": 10833.3770185709, "calibration_cpu_ns": 859166.0, "calibration_gpu_ticks": 859166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9917.0, "compute_span_ns": 9917.0, "command_buffer_ns": 9916.664566844702, "calibration_cpu_ns": 559542.0, "calibration_gpu_ticks": 559542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11334.0, "compute_span_ns": 11334.0, "command_buffer_ns": 11374.999303370714, "calibration_cpu_ns": 769625.0, "calibration_gpu_ticks": 769625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9584.0, "compute_span_ns": 9584.0, "command_buffer_ns": 9624.985978007317, "calibration_cpu_ns": 392666.0, "calibration_gpu_ticks": 392666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 30375.0, "compute_span_ns": 30375.0, "command_buffer_ns": 30374.969355762005, "calibration_cpu_ns": 1551833.0, "calibration_gpu_ticks": 1551833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10708.0, "compute_span_ns": 10708.0, "command_buffer_ns": 10708.346962928772, "calibration_cpu_ns": 837708.0, "calibration_gpu_ticks": 837708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10084.0, "compute_span_ns": 10084.0, "command_buffer_ns": 10124.989785254002, "calibration_cpu_ns": 392375.0, "calibration_gpu_ticks": 392375.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 7.727203125, 9.0423125, 7.611328125, 10.260421875, 7.440109375, 8.201828125, 8.84765625, 7.7109375, 7.425125 ], "compute_throughput_us_p50": 7.727203125, "compute_throughput_us_p90": 9.285934375, "command_buffer_throughput_us": [ 7.726562216703314, 9.042317287821788, 7.6113283284939826, 10.260417184326798, 7.440755325660575, 8.202473509300034, 8.847656317811925, 7.710937097726855, 7.424479008477647 ], "command_buffer_throughput_us_p50": 7.726562216703314, "command_buffer_throughput_us_p90": 9.28593726712279, "compute_latency_us": [ 11.125, 9.875, 10.833, 9.917, 11.334, 9.584, 30.375, 10.708, 10.084 ], "compute_latency_us_p50": 10.708, "compute_latency_us_p90": 15.142200000000003, "command_buffer_latency_us": [ 11.124997399747372, 9.875046089291573, 10.8333770185709, 9.916664566844702, 11.374999303370714, 9.624985978007317, 30.374969355762005, 10.708346962928772, 10.124989785254002 ], "command_buffer_latency_us_p50": 10.708346962928772, "command_buffer_latency_us_p90": 15.174993313848976, "counter_control_throughput_ratio": 0.8318049712901512, "counter_control_latency_ratio": 0.9390212045839154 }, "download_ms": 0.334375, "allocation_upload_ms": 1.717792, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-a0k85gr_/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "53243cd32bb4620fa91e934e3d500807c417fa70fc3ac1977dac07575a48be10", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.180833, "capture_ms": 0.083125, "compile_ms": 73.610292, "allocation_upload_ms": 1.534625, "cold_call_ms": 1.92825, "warmup_ms": 200.118625, "download_ms": 0.321292, "repetitions": 2716, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.74679381443, 6.7874326215, 7.83282731959, 6.75455633284, 6.80605670103, 7.75024558174, 6.78140353461, 6.75541568483, 7.64370103093 ], "latency_us": [ 542.667, 217.416, 213.584, 221.583, 290.083, 209.125, 212.292, 213.416, 208.541 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 364042, "compute_span_ns": 364042, "calibration_cpu_ns": 2017417, "calibration_gpu_ticks": 2017417, "compute_passes": 1, "command_buffer_ns": 364125.007764, "command_buffers": 2 }, { "compute_ns": 294500, "compute_span_ns": 294500, "calibration_cpu_ns": 797500, "calibration_gpu_ticks": 797500, "compute_passes": 1, "command_buffer_ns": 294625.002425, "command_buffers": 2 }, { "compute_ns": 311667, "compute_span_ns": 311667, "calibration_cpu_ns": 743833, "calibration_gpu_ticks": 743833, "compute_passes": 1, "command_buffer_ns": 311791.722197, "command_buffers": 2 }, { "compute_ns": 295958, "compute_span_ns": 295958, "calibration_cpu_ns": 695959, "calibration_gpu_ticks": 695959, "compute_passes": 1, "command_buffer_ns": 295958.365314, "command_buffers": 2 }, { "compute_ns": 295208, "compute_span_ns": 295208, "calibration_cpu_ns": 785292, "calibration_gpu_ticks": 785292, "compute_passes": 1, "command_buffer_ns": 295208.359603, "command_buffers": 2 }, { "compute_ns": 761209, "compute_span_ns": 761209, "calibration_cpu_ns": 2224417, "calibration_gpu_ticks": 2224417, "compute_passes": 1, "command_buffer_ns": 761249.975767, "command_buffers": 2 }, { "compute_ns": 733375, "compute_span_ns": 733375, "calibration_cpu_ns": 2081750, "calibration_gpu_ticks": 2081750, "compute_passes": 1, "command_buffer_ns": 733458.32061, "command_buffers": 2 }, { "compute_ns": 289334, "compute_span_ns": 289334, "calibration_cpu_ns": 707708, "calibration_gpu_ticks": 707708, "compute_passes": 1, "command_buffer_ns": 289416.697342, "command_buffers": 2 }, { "compute_ns": 294625, "compute_span_ns": 294625, "calibration_cpu_ns": 802292, "calibration_gpu_ticks": 802292, "compute_passes": 1, "command_buffer_ns": 294625.002425, "command_buffers": 2 } ], "latency": [ { "compute_ns": 14083, "compute_span_ns": 14083, "calibration_cpu_ns": 672875, "calibration_gpu_ticks": 672875, "compute_passes": 1, "command_buffer_ns": 14083.2853504, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 439584, "calibration_gpu_ticks": 439584, "compute_passes": 1, "command_buffer_ns": 7083.29025656, "command_buffers": 2 }, { "compute_ns": 8500, "compute_span_ns": 8500, "calibration_cpu_ns": 833209, "calibration_gpu_ticks": 833209, "compute_passes": 1, "command_buffer_ns": 8583.35988596, "command_buffers": 2 }, { "compute_ns": 9833, "compute_span_ns": 9833, "calibration_cpu_ns": 447334, "calibration_gpu_ticks": 447334, "compute_passes": 1, "command_buffer_ns": 9874.98788163, "command_buffers": 2 }, { "compute_ns": 8666, "compute_span_ns": 8666, "calibration_cpu_ns": 863166, "calibration_gpu_ticks": 863166, "compute_passes": 1, "command_buffer_ns": 8666.65504873, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 484084, "calibration_gpu_ticks": 484084, "compute_passes": 1, "command_buffer_ns": 7083.34846422, "command_buffers": 2 }, { "compute_ns": 7875, "compute_span_ns": 7875, "calibration_cpu_ns": 631458, "calibration_gpu_ticks": 631458, "compute_passes": 1, "command_buffer_ns": 7875.0308603, "command_buffers": 2 }, { "compute_ns": 7625, "compute_span_ns": 7625, "calibration_cpu_ns": 484833, "calibration_gpu_ticks": 484833, "compute_passes": 1, "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "compute_ns": 9209, "compute_span_ns": 9209, "calibration_cpu_ns": 825417, "calibration_gpu_ticks": 825417, "compute_passes": 1, "command_buffer_ns": 9250.0122264, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 671000.045259, "command_buffers": 2 }, { "command_buffer_ns": 296250.043903, "command_buffers": 2 }, { "command_buffer_ns": 703624.973539, "command_buffers": 2 }, { "command_buffer_ns": 687208.375894, "command_buffers": 2 }, { "command_buffer_ns": 295249.97808, "command_buffers": 2 }, { "command_buffer_ns": 295749.981888, "command_buffers": 2 }, { "command_buffer_ns": 319875.020068, "command_buffers": 2 }, { "command_buffer_ns": 312999.996822, "command_buffers": 2 }, { "command_buffer_ns": 684999.977238, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 87624.9978319, "command_buffers": 2 }, { "command_buffer_ns": 7624.97074902, "command_buffers": 2 }, { "command_buffer_ns": 33083.2554027, "command_buffers": 2 }, { "command_buffer_ns": 7583.35227147, "command_buffers": 2 }, { "command_buffer_ns": 114208.320156, "command_buffers": 2 }, { "command_buffer_ns": 7625.02895668, "command_buffers": 2 }, { "command_buffer_ns": 59708.3126195, "command_buffers": 2 }, { "command_buffer_ns": 7583.35227147, "command_buffers": 2 }, { "command_buffer_ns": 52166.6370332, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 10.484375707171875, 4.628906935984375, 10.994140211546876, 10.73763087334375, 4.613280907499999, 4.621093467, 4.9980471885625, 4.89062495034375, 10.70312464434375 ], "command_buffer_throughput_us_p50": 4.9980471885625, "command_buffer_throughput_us_p90": 10.788932740984375, "command_buffer_latency_us": [ 87.62499783189999, 7.62497074902, 33.0832554027, 7.58335227147, 114.208320156, 7.6250289566800005, 59.7083126195, 7.58335227147, 52.1666370332 ], "command_buffer_latency_us_p50": 33.0832554027, "command_buffer_latency_us_p90": 92.94166229672 }, "compute_throughput_us": [ 5.68815625, 4.6015625, 4.869796875, 4.62434375, 4.612625, 11.893890625, 11.458984375, 4.52084375, 4.603515625 ], "compute_throughput_us_p50": 4.62434375, "compute_throughput_us_p90": 11.545965625, "command_buffer_throughput_us": [ 5.6894532463125, 4.603515662890625, 4.871745659328125, 4.62434945803125, 4.612630618796874, 11.894530871359375, 11.460286259531252, 4.52213589596875, 4.603515662890625 ], "command_buffer_throughput_us_p50": 4.62434945803125, "command_buffer_throughput_us_p90": 11.547135181896877, "compute_latency_us": [ 14.083, 7.042, 8.5, 9.833, 8.666, 7.042, 7.875, 7.625, 9.209 ], "compute_latency_us_p50": 8.5, "compute_latency_us_p90": 10.683000000000002, "command_buffer_latency_us": [ 14.083285350399999, 7.08329025656, 8.58335988596, 9.87498788163, 8.66665504873, 7.08334846422, 7.8750308603, 7.708324119449999, 9.2500122264 ], "command_buffer_latency_us_p50": 8.58335988596, "command_buffer_latency_us_p90": 10.716647375384001, "counter_control_throughput_ratio": 0.9246539945065508, "counter_control_latency_ratio": 0.25944725757730275 }, "correctness": { "max_abs_error": 2.1991937471099732e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 6.7874326215, "throughput_us_p90": 7.766761929309999, "latency_us_p50": 213.584, "latency_us_p90": 340.5998000000001 }, "slowdown": 0.4526725694942879, "device_compute_slowdown": 0.5984498757433662, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-mbsja9tt/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b8d9d984f3aaf8bfdb9269f6e0efc630605362c6ff4e852cc112881dfaf5e97b", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.488875, "capture_ms": 0.089667, "compile_ms": 65.189875, "allocation_upload_ms": 4.017042, "cold_call_ms": 9.64275, "warmup_ms": 200.395791, "download_ms": 0.835458, "repetitions": 1510, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 23.5842993377, 23.019894702, 23.0344099338, 23.5717437086, 24.5195364238, 23.0382172185, 23.0501927152, 24.7188741722, 23.0996139073 ], "latency_us": [ 269.75, 256.75, 252, 224.125, 234.209, 225.333, 221.417, 247.5, 246.167 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1522083, "compute_span_ns": 1522083, "calibration_cpu_ns": 2702875, "calibration_gpu_ticks": 2702875, "compute_passes": 1, "command_buffer_ns": 1522124.97778, "command_buffers": 2 }, { "compute_ns": 1041625, "compute_span_ns": 1041625, "calibration_cpu_ns": 1572792, "calibration_gpu_ticks": 1572792, "compute_passes": 1, "command_buffer_ns": 1041708.33947, "command_buffers": 2 }, { "compute_ns": 1440500, "compute_span_ns": 1440500, "calibration_cpu_ns": 2676708, "calibration_gpu_ticks": 2676708, "compute_passes": 1, "command_buffer_ns": 1440625.05569, "command_buffers": 2 }, { "compute_ns": 1064750, "compute_span_ns": 1064750, "calibration_cpu_ns": 1557916, "calibration_gpu_ticks": 1557916, "compute_passes": 1, "command_buffer_ns": 1064791.69335, "command_buffers": 2 }, { "compute_ns": 1577125, "compute_span_ns": 1577125, "calibration_cpu_ns": 2904083, "calibration_gpu_ticks": 2904083, "compute_passes": 1, "command_buffer_ns": 1577124.98913, "command_buffers": 2 }, { "compute_ns": 1032584, "compute_span_ns": 1032584, "calibration_cpu_ns": 1583000, "calibration_gpu_ticks": 1583000, "compute_passes": 1, "command_buffer_ns": 1032666.71067, "command_buffers": 2 }, { "compute_ns": 1541792, "compute_span_ns": 1541792, "calibration_cpu_ns": 3072208, "calibration_gpu_ticks": 3072208, "compute_passes": 1, "command_buffer_ns": 1541833.33507, "command_buffers": 2 }, { "compute_ns": 1022042, "compute_span_ns": 1022042, "calibration_cpu_ns": 1465666, "calibration_gpu_ticks": 1465666, "compute_passes": 1, "command_buffer_ns": 1022124.95403, "command_buffers": 2 }, { "compute_ns": 1562250, "compute_span_ns": 1562250, "calibration_cpu_ns": 2769375, "calibration_gpu_ticks": 2769375, "compute_passes": 1, "command_buffer_ns": 1562374.99323, "command_buffers": 2 } ], "latency": [ { "compute_ns": 313125, "compute_span_ns": 313125, "calibration_cpu_ns": 1906708, "calibration_gpu_ticks": 1906708, "compute_passes": 1, "command_buffer_ns": 313250.056934, "command_buffers": 2 }, { "compute_ns": 22959, "compute_span_ns": 22959, "calibration_cpu_ns": 862625, "calibration_gpu_ticks": 862625, "compute_passes": 1, "command_buffer_ns": 23083.2956731, "command_buffers": 2 }, { "compute_ns": 99375, "compute_span_ns": 99375, "calibration_cpu_ns": 1657708, "calibration_gpu_ticks": 1657708, "compute_passes": 1, "command_buffer_ns": 99416.6475721, "command_buffers": 2 }, { "compute_ns": 20500, "compute_span_ns": 20500, "calibration_cpu_ns": 852916, "calibration_gpu_ticks": 852916, "compute_passes": 1, "command_buffer_ns": 20541.6581593, "command_buffers": 2 }, { "compute_ns": 86209, "compute_span_ns": 86209, "calibration_cpu_ns": 1485291, "calibration_gpu_ticks": 1485291, "compute_passes": 1, "command_buffer_ns": 86375.0465214, "command_buffers": 2 }, { "compute_ns": 20083, "compute_span_ns": 20083, "calibration_cpu_ns": 808292, "calibration_gpu_ticks": 808292, "compute_passes": 1, "command_buffer_ns": 20083.389245, "command_buffers": 2 }, { "compute_ns": 70875, "compute_span_ns": 70875, "calibration_cpu_ns": 1396500, "calibration_gpu_ticks": 1396500, "compute_passes": 1, "command_buffer_ns": 70874.9867044, "command_buffers": 2 }, { "compute_ns": 19416, "compute_span_ns": 19416, "calibration_cpu_ns": 677791, "calibration_gpu_ticks": 677791, "compute_passes": 1, "command_buffer_ns": 19458.2971744, "command_buffers": 2 }, { "compute_ns": 20167, "compute_span_ns": 20167, "calibration_cpu_ns": 437667, "calibration_gpu_ticks": 437667, "compute_passes": 1, "command_buffer_ns": 20291.7144634, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1242958.35849, "command_buffers": 2 }, { "command_buffer_ns": 1468916.71466, "command_buffers": 2 }, { "command_buffer_ns": 1030249.92859, "command_buffers": 2 }, { "command_buffer_ns": 1579249.9762, "command_buffers": 2 }, { "command_buffer_ns": 1031874.97007, "command_buffers": 2 }, { "command_buffer_ns": 1489208.37091, "command_buffers": 2 }, { "command_buffer_ns": 1036083.38395, "command_buffers": 2 }, { "command_buffer_ns": 1423833.30967, "command_buffers": 2 }, { "command_buffer_ns": 1053749.99112, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 17874.9323823, "command_buffers": 2 }, { "command_buffer_ns": 17708.3420567, "command_buffers": 2 }, { "command_buffer_ns": 20041.7125598, "command_buffers": 2 }, { "command_buffer_ns": 21000.043489, "command_buffers": 2 }, { "command_buffer_ns": 20041.6543521, "command_buffers": 2 }, { "command_buffer_ns": 17916.6672751, "command_buffers": 2 }, { "command_buffer_ns": 20833.3367482, "command_buffers": 2 }, { "command_buffer_ns": 18083.3158083, "command_buffers": 2 }, { "command_buffer_ns": 20375.0096262, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 19.42122435140625, 22.9518236665625, 16.09765513421875, 24.675780878125, 16.12304640734375, 23.26888079546875, 16.18880287421875, 22.24739546359375, 16.46484361125 ], "command_buffer_throughput_us_p50": 19.42122435140625, "command_buffer_throughput_us_p90": 23.550260812, "command_buffer_latency_us": [ 17.874932382300003, 17.708342056699998, 20.041712559799997, 21.000043489, 20.0416543521, 17.9166672751, 20.8333367482, 18.0833158083, 20.3750096262 ], "command_buffer_latency_us_p50": 20.0416543521, "command_buffer_latency_us_p90": 20.86667809636 }, "compute_throughput_us": [ 23.782546875, 16.275390625, 22.5078125, 16.63671875, 24.642578125, 16.134125, 24.0905, 15.96940625, 24.41015625 ], "compute_throughput_us_p50": 22.5078125, "compute_throughput_us_p90": 24.456640625, "command_buffer_throughput_us": [ 23.7832027778125, 16.27669280421875, 22.50976649515625, 16.63737020859375, 24.64257795515625, 16.13541735421875, 24.09114586046875, 15.97070240671875, 24.41210926921875 ], "command_buffer_throughput_us_p50": 22.50976649515625, "command_buffer_throughput_us_p90": 24.45820300640625, "compute_latency_us": [ 313.125, 22.959, 99.375, 20.5, 86.209, 20.083, 70.875, 19.416, 20.167 ], "compute_latency_us_p50": 22.959, "compute_latency_us_p90": 142.12500000000003, "command_buffer_latency_us": [ 313.250056934, 23.0832956731, 99.4166475721, 20.541658159300002, 86.3750465214, 20.083389245, 70.8749867044, 19.4582971744, 20.291714463399998 ], "command_buffer_latency_us_p50": 23.0832956731, "command_buffer_latency_us_p90": 142.18332944448002, "counter_control_throughput_ratio": 1.2245985292935668, "counter_control_latency_ratio": 1.303526642934163 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 23.0996139073, "throughput_us_p90": 24.559403973480002, "latency_us_p50": 246.167, "latency_us_p90": 259.35 }, "torch": { "cold_call_ms": 7.703083, "warmup_ms": 200.013542, "repetitions": 398, "throughput_us": [ 63.876673366834176, 62.52334673366833, 63.26654020100502, 62.388716080402006, 63.093175879396995, 62.76570351758794, 62.87091708542714, 62.65117085427135, 62.82715577889447 ], "latency_us": [ 415.375, 307.417, 296.125, 2147.417, 671.2909999999999, 326.41700000000003, 212.875, 625.541, 325.834 ], "throughput_us_p50": 62.82715577889447, "throughput_us_p90": 63.38856683417085, "latency_us_p50": 326.41700000000003, "latency_us_p90": 966.5162000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2589749.9872371554, "command_buffers": 1 }, { "command_buffer_ns": 2783125.0336021185, "command_buffers": 1 }, { "command_buffer_ns": 2921416.6570454836, "command_buffers": 1 }, { "command_buffer_ns": 2768874.983303249, "command_buffers": 1 }, { "command_buffer_ns": 2729624.9754726887, "command_buffers": 1 }, { "command_buffer_ns": 2806958.3349861205, "command_buffers": 1 }, { "command_buffer_ns": 2755749.970674515, "command_buffers": 1 }, { "command_buffer_ns": 2808208.344504237, "command_buffers": 1 }, { "command_buffer_ns": 2630958.3336114883, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 107041.67652875185, "command_buffers": 1 }, { "command_buffer_ns": 217875.00008940697, "command_buffers": 1 }, { "command_buffer_ns": 44125.01584738493, "command_buffers": 1 }, { "command_buffer_ns": 129000.05094707012, "command_buffers": 1 }, { "command_buffer_ns": 43999.9857917428, "command_buffers": 1 }, { "command_buffer_ns": 73458.35911110044, "command_buffers": 1 }, { "command_buffer_ns": 43583.3353549242, "command_buffers": 1 }, { "command_buffer_ns": 219708.30857753754, "command_buffers": 1 }, { "command_buffer_ns": 44208.3110101521, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 40.464843550580554, 43.4863286500331, 45.64713526633568, 43.263671614113264, 42.65039024176076, 43.85872398415813, 43.05859329178929, 43.878255382878706, 41.108723962679505 ], "command_buffer_throughput_us_p50": 43.263671614113264, "command_buffer_throughput_us_p90": 44.2320313595701, "command_buffer_latency_us": [ 107.04167652875185, 217.87500008940697, 44.12501584738493, 129.00005094707012, 43.9999857917428, 73.45835911110044, 43.5833353549242, 219.70830857753754, 44.2083110101521 ], "command_buffer_latency_us_p50": 73.45835911110044, "command_buffer_latency_us_p90": 218.24166178703308 }, "throughput": [ { "compute_ns": 2754500.0, "compute_span_ns": 2754500.0, "command_buffer_ns": 2754500.019364059, "calibration_cpu_ns": 3964708.0, "calibration_gpu_ticks": 3964708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2741667.0, "compute_span_ns": 2741667.0, "command_buffer_ns": 2741666.6853241622, "calibration_cpu_ns": 4163625.0, "calibration_gpu_ticks": 4163625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2759292.0, "compute_span_ns": 2759292.0, "command_buffer_ns": 2759333.3506956697, "calibration_cpu_ns": 4153459.0, "calibration_gpu_ticks": 4153459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2653458.0, "compute_span_ns": 2653458.0, "command_buffer_ns": 2653416.7118370533, "calibration_cpu_ns": 4039416.0, "calibration_gpu_ticks": 4039416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2704459.0, "compute_span_ns": 2704459.0, "command_buffer_ns": 2704499.987885356, "calibration_cpu_ns": 4322417.0, "calibration_gpu_ticks": 4322417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2746083.0, "compute_span_ns": 2746083.0, "command_buffer_ns": 2746041.6313260794, "calibration_cpu_ns": 4216333.0, "calibration_gpu_ticks": 4216333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2733917.0, "compute_span_ns": 2733917.0, "command_buffer_ns": 2733958.3029970527, "calibration_cpu_ns": 4249166.0, "calibration_gpu_ticks": 4249166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2776959.0, "compute_span_ns": 2776959.0, "command_buffer_ns": 2776999.957859516, "calibration_cpu_ns": 4287792.0, "calibration_gpu_ticks": 4287792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2696292.0, "compute_span_ns": 2696292.0, "command_buffer_ns": 2696291.6599586606, "calibration_cpu_ns": 4134667.0, "calibration_gpu_ticks": 4134667.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 43375.0, "compute_span_ns": 43375.0, "command_buffer_ns": 43375.0101365149, "calibration_cpu_ns": 936583.0, "calibration_gpu_ticks": 936583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43250.0, "compute_span_ns": 43250.0, "command_buffer_ns": 43250.03828853369, "calibration_cpu_ns": 542250.0, "calibration_gpu_ticks": 542250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43875.0, "compute_span_ns": 43875.0, "command_buffer_ns": 43875.01394376159, "calibration_cpu_ns": 621959.0, "calibration_gpu_ticks": 621959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44250.0, "compute_span_ns": 44250.0, "command_buffer_ns": 44250.04590302706, "calibration_cpu_ns": 481958.0, "calibration_gpu_ticks": 481958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 42583.0, "compute_span_ns": 42583.0, "command_buffer_ns": 42541.65105521679, "calibration_cpu_ns": 560291.0, "calibration_gpu_ticks": 560291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46708.0, "compute_span_ns": 46708.0, "command_buffer_ns": 46666.653361171484, "calibration_cpu_ns": 622000.0, "calibration_gpu_ticks": 622000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 42625.0, "compute_span_ns": 42625.0, "command_buffer_ns": 42625.004425644875, "calibration_cpu_ns": 574792.0, "calibration_gpu_ticks": 574792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44208.0, "compute_span_ns": 44208.0, "command_buffer_ns": 44166.69253259897, "calibration_cpu_ns": 494958.0, "calibration_gpu_ticks": 494958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44708.0, "compute_span_ns": 44708.0, "command_buffer_ns": 44666.69633984566, "calibration_cpu_ns": 661375.0, "calibration_gpu_ticks": 661375.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 43.0390625, 42.838546875, 43.1139375, 41.46028125, 42.257171875, 42.907546875, 42.717453125, 43.389984375, 42.1295625 ], "compute_throughput_us_p50": 42.838546875, "compute_throughput_us_p90": 43.169146874999996, "command_buffer_throughput_us": [ 43.03906280256342, 42.838541958190035, 43.11458360461984, 41.45963612245396, 42.25781231070869, 42.90690048946999, 42.71809848432895, 43.39062434155494, 42.12955718685407 ], "command_buffer_throughput_us_p50": 42.838541958190035, "command_buffer_throughput_us_p90": 43.16979175200686, "compute_latency_us": [ 43.375, 43.25, 43.875, 44.25, 42.583, 46.708, 42.625, 44.208, 44.708 ], "compute_latency_us_p50": 43.875, "compute_latency_us_p90": 45.108, "command_buffer_latency_us": [ 43.3750101365149, 43.25003828853369, 43.87501394376159, 44.25004590302706, 42.54165105521679, 46.666653361171484, 42.625004425644875, 44.16669253259897, 44.66669633984566 ], "command_buffer_latency_us_p50": 43.87501394376159, "command_buffer_latency_us_p90": 45.06668774411082, "counter_control_throughput_ratio": 0.9888867267609268, "counter_control_latency_ratio": 0.6352803673519518 }, "download_ms": 0.410458, "allocation_upload_ms": 1.439875, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3676692605438595, "device_compute_slowdown": 0.5254102704668364, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.433792, "warmup_ms": 200.612167, "repetitions": 543, "throughput_us": [ 58.96439594843463, 61.14901841620625, 59.85366850828729, 59.81752670349908, 60.776012891344386, 61.00805709023941, 61.617327808471444, 61.37453959484346, 61.03130755064457 ], "latency_us": [ 417.084, 302.70799999999997, 296.08299999999997, 306.625, 2074.833, 1234.5410000000002, 726.6669999999999, 555.292, 357.666 ], "throughput_us_p50": 61.00805709023941, "throughput_us_p90": 61.42309723756906, "latency_us_p50": 417.084, "latency_us_p90": 1402.5994000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3098374.989349395, "command_buffers": 1 }, { "command_buffer_ns": 2713541.6748933494, "command_buffers": 1 }, { "command_buffer_ns": 2707124.9787695706, "command_buffers": 1 }, { "command_buffer_ns": 2758041.664492339, "command_buffers": 1 }, { "command_buffer_ns": 2762125.0483207405, "command_buffers": 1 }, { "command_buffer_ns": 2638958.3363197744, "command_buffers": 1 }, { "command_buffer_ns": 2819499.99043718, "command_buffers": 1 }, { "command_buffer_ns": 2683250.0007003546, "command_buffers": 1 }, { "command_buffer_ns": 4864916.671067476, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 47000.00863522291, "command_buffers": 1 }, { "command_buffer_ns": 43499.98198449612, "command_buffers": 1 }, { "command_buffer_ns": 42249.972466379404, "command_buffers": 1 }, { "command_buffer_ns": 44125.01584738493, "command_buffers": 1 }, { "command_buffer_ns": 44124.957639724016, "command_buffers": 1 }, { "command_buffer_ns": 43875.01394376159, "command_buffers": 1 }, { "command_buffer_ns": 46666.653361171484, "command_buffers": 1 }, { "command_buffer_ns": 42625.004425644875, "command_buffers": 1 }, { "command_buffer_ns": 43500.04019215703, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 48.4121092085843, 42.399088670208585, 42.29882779327454, 43.094401007692795, 43.15820388001157, 41.233724004996475, 44.054687350580934, 41.92578126094304, 76.01432298542932 ], "command_buffer_throughput_us_p50": 43.094401007692795, "command_buffer_throughput_us_p90": 53.93255196395331, "command_buffer_latency_us": [ 47.00000863522291, 43.49998198449612, 42.249972466379404, 44.12501584738493, 44.124957639724016, 43.87501394376159, 46.666653361171484, 42.625004425644875, 43.50004019215703 ], "command_buffer_latency_us_p50": 43.87501394376159, "command_buffer_latency_us_p90": 46.73332441598177 }, "throughput": [ { "compute_ns": 2578209.0, "compute_span_ns": 2578209.0, "command_buffer_ns": 2578249.9578781426, "calibration_cpu_ns": 5266791.0, "calibration_gpu_ticks": 5266791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2759833.0, "compute_span_ns": 2759833.0, "command_buffer_ns": 2759791.6778177023, "calibration_cpu_ns": 4222042.0, "calibration_gpu_ticks": 4222042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2601500.0, "compute_span_ns": 2601500.0, "command_buffer_ns": 2601500.0184997916, "calibration_cpu_ns": 4579041.0, "calibration_gpu_ticks": 4579041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2670000.0, "compute_span_ns": 2670000.0, "command_buffer_ns": 2670000.0162236392, "calibration_cpu_ns": 4320500.0, "calibration_gpu_ticks": 4320500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2727875.0, "compute_span_ns": 2727875.0, "command_buffer_ns": 2727875.020354986, "calibration_cpu_ns": 3981333.0, "calibration_gpu_ticks": 3981333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2663875.0, "compute_span_ns": 2663875.0, "command_buffer_ns": 2663874.998688698, "calibration_cpu_ns": 4470916.0, "calibration_gpu_ticks": 4470916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2769500.0, "compute_span_ns": 2769500.0, "command_buffer_ns": 2769499.958958477, "calibration_cpu_ns": 4521625.0, "calibration_gpu_ticks": 4521625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2651083.0, "compute_span_ns": 2651083.0, "command_buffer_ns": 2651041.722856462, "calibration_cpu_ns": 5038209.0, "calibration_gpu_ticks": 5038209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2611041.0, "compute_span_ns": 2611041.0, "command_buffer_ns": 2611000.0326298177, "calibration_cpu_ns": 3588791.0, "calibration_gpu_ticks": 3588791.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 44292.0, "compute_span_ns": 44292.0, "command_buffer_ns": 44291.66438058019, "calibration_cpu_ns": 526291.0, "calibration_gpu_ticks": 526291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 113125.0, "compute_span_ns": 113125.0, "command_buffer_ns": 113125.01737847924, "calibration_cpu_ns": 1533292.0, "calibration_gpu_ticks": 1533292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44250.0, "compute_span_ns": 44250.0, "command_buffer_ns": 44249.987695366144, "calibration_cpu_ns": 469667.0, "calibration_gpu_ticks": 469667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 173625.0, "compute_span_ns": 173625.0, "command_buffer_ns": 173625.01239404082, "calibration_cpu_ns": 1830250.0, "calibration_gpu_ticks": 1830250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44084.0, "compute_span_ns": 44084.0, "command_buffer_ns": 44125.01584738493, "calibration_cpu_ns": 521292.0, "calibration_gpu_ticks": 521292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47167.0, "compute_span_ns": 47167.0, "command_buffer_ns": 47166.65716841817, "calibration_cpu_ns": 1047584.0, "calibration_gpu_ticks": 1047584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44375.0, "compute_span_ns": 44375.0, "command_buffer_ns": 44375.01775100827, "calibration_cpu_ns": 710500.0, "calibration_gpu_ticks": 710500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45542.0, "compute_span_ns": 45542.0, "command_buffer_ns": 45541.6738986969, "calibration_cpu_ns": 954500.0, "calibration_gpu_ticks": 954500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44084.0, "compute_span_ns": 44084.0, "command_buffer_ns": 44125.01584738493, "calibration_cpu_ns": 482083.0, "calibration_gpu_ticks": 482083.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 40.284515625, 43.122390625, 40.6484375, 41.71875, 42.623046875, 41.623046875, 43.2734375, 41.423171875, 40.797515625 ], "compute_throughput_us_p50": 41.623046875, "compute_throughput_us_p90": 43.1526, "command_buffer_throughput_us": [ 40.28515559184598, 43.1217449659016, 40.648437789059244, 41.71875025349436, 42.62304719304666, 41.6230468545109, 43.2734368587262, 41.42252691963222, 40.7968755098409 ], "command_buffer_throughput_us_p50": 41.6230468545109, "command_buffer_throughput_us_p90": 43.15208334446652, "compute_latency_us": [ 44.292, 113.125, 44.25, 173.625, 44.084, 47.167, 44.375, 45.542, 44.084 ], "compute_latency_us_p50": 44.375, "compute_latency_us_p90": 125.22500000000001, "command_buffer_latency_us": [ 44.29166438058019, 113.12501737847924, 44.249987695366144, 173.62501239404082, 44.12501584738493, 47.16665716841817, 44.37501775100827, 45.5416738986969, 44.12501584738493 ], "command_buffer_latency_us_p50": 44.37501775100827, "command_buffer_latency_us_p90": 125.22501638159157, "counter_control_throughput_ratio": 0.982266348058774, "counter_control_latency_ratio": 1.0473376694050691 }, "download_ms": 0.398834, "allocation_upload_ms": 3.208333, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-zrf8seum/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "83a70ba0e4d1dab6671da56395e0b13750f8d9c9277e4782c4e371450de131c7", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 23.060125, "capture_ms": 0.075875, "compile_ms": 64.325333, "allocation_upload_ms": 2.165417, "cold_call_ms": 5.454792, "warmup_ms": 200.359709, "download_ms": 0.660667, "repetitions": 1462, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 23.4330827633, 23.4223098495, 23.3282318741, 23.331880301, 23.5616737346, 24.0980677155, 24.0816805746, 23.3315943912, 23.3718932969 ], "latency_us": [ 283.625, 263.584, 250.833, 225.625, 234.875, 235.583, 237.834, 268, 2415.541 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1038125, "compute_span_ns": 1038125, "calibration_cpu_ns": 1522667, "calibration_gpu_ticks": 1522667, "compute_passes": 1, "command_buffer_ns": 1038208.31282, "command_buffers": 2 }, { "compute_ns": 1494625, "compute_span_ns": 1494625, "calibration_cpu_ns": 2786500, "calibration_gpu_ticks": 2786500, "compute_passes": 1, "command_buffer_ns": 1494625.00121, "command_buffers": 2 }, { "compute_ns": 1066250, "compute_span_ns": 1066250, "calibration_cpu_ns": 1554833, "calibration_gpu_ticks": 1554833, "compute_passes": 1, "command_buffer_ns": 1066374.99994, "command_buffers": 2 }, { "compute_ns": 1507250, "compute_span_ns": 1507250, "calibration_cpu_ns": 2877083, "calibration_gpu_ticks": 2877083, "compute_passes": 1, "command_buffer_ns": 1507291.68672, "command_buffers": 2 }, { "compute_ns": 1033375, "compute_span_ns": 1033375, "calibration_cpu_ns": 1626250, "calibration_gpu_ticks": 1626250, "compute_passes": 1, "command_buffer_ns": 1033458.27665, "command_buffers": 2 }, { "compute_ns": 1480000, "compute_span_ns": 1480000, "calibration_cpu_ns": 2371250, "calibration_gpu_ticks": 2371250, "compute_passes": 1, "command_buffer_ns": 1480125.06543, "command_buffers": 2 }, { "compute_ns": 1044333, "compute_span_ns": 1044333, "calibration_cpu_ns": 1531291, "calibration_gpu_ticks": 1531291, "compute_passes": 1, "command_buffer_ns": 1044375.00704, "command_buffers": 2 }, { "compute_ns": 1107083, "compute_span_ns": 1107083, "calibration_cpu_ns": 3259042, "calibration_gpu_ticks": 3259042, "compute_passes": 1, "command_buffer_ns": 1107166.69587, "command_buffers": 2 }, { "compute_ns": 1027000, "compute_span_ns": 1027000, "calibration_cpu_ns": 1744000, "calibration_gpu_ticks": 1744000, "compute_passes": 1, "command_buffer_ns": 1027041.69694, "command_buffers": 2 } ], "latency": [ { "compute_ns": 22583, "compute_span_ns": 22583, "calibration_cpu_ns": 799083, "calibration_gpu_ticks": 799083, "compute_passes": 1, "command_buffer_ns": 22708.3801292, "command_buffers": 2 }, { "compute_ns": 17959, "compute_span_ns": 17959, "calibration_cpu_ns": 444334, "calibration_gpu_ticks": 444334, "compute_passes": 1, "command_buffer_ns": 18041.6973308, "command_buffers": 2 }, { "compute_ns": 22250, "compute_span_ns": 22250, "calibration_cpu_ns": 1136042, "calibration_gpu_ticks": 1136042, "compute_passes": 1, "command_buffer_ns": 22291.6714847, "command_buffers": 2 }, { "compute_ns": 17958, "compute_span_ns": 17958, "calibration_cpu_ns": 426416, "calibration_gpu_ticks": 426416, "compute_passes": 1, "command_buffer_ns": 17958.3439603, "command_buffers": 2 }, { "compute_ns": 20291, "compute_span_ns": 20291, "calibration_cpu_ns": 874625, "calibration_gpu_ticks": 874625, "compute_passes": 1, "command_buffer_ns": 20333.2747333, "command_buffers": 2 }, { "compute_ns": 18083, "compute_span_ns": 18083, "calibration_cpu_ns": 524250, "calibration_gpu_ticks": 524250, "compute_passes": 1, "command_buffer_ns": 18208.2876563, "command_buffers": 2 }, { "compute_ns": 21791, "compute_span_ns": 21791, "calibration_cpu_ns": 847792, "calibration_gpu_ticks": 847792, "compute_passes": 1, "command_buffer_ns": 21874.9628402, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 408250, "calibration_gpu_ticks": 408250, "compute_passes": 1, "command_buffer_ns": 18541.701138, "command_buffers": 2 }, { "compute_ns": 22375, "compute_span_ns": 22375, "calibration_cpu_ns": 904500, "calibration_gpu_ticks": 904500, "compute_passes": 1, "command_buffer_ns": 22458.2618102, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1266624.95313, "command_buffers": 2 }, { "command_buffer_ns": 1036041.64906, "command_buffers": 2 }, { "command_buffer_ns": 1424291.695, "command_buffers": 2 }, { "command_buffer_ns": 1057874.93521, "command_buffers": 2 }, { "command_buffer_ns": 1498208.26482, "command_buffers": 2 }, { "command_buffer_ns": 1059208.2981, "command_buffers": 2 }, { "command_buffer_ns": 1329791.67392, "command_buffers": 2 }, { "command_buffer_ns": 1033374.98149, "command_buffers": 2 }, { "command_buffer_ns": 1954208.30278, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 65000.0292808, "command_buffers": 2 }, { "command_buffer_ns": 19999.9776669, "command_buffers": 2 }, { "command_buffer_ns": 94750.0229813, "command_buffers": 2 }, { "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "command_buffer_ns": 84458.2682475, "command_buffers": 2 }, { "command_buffer_ns": 20249.9795705, "command_buffers": 2 }, { "command_buffer_ns": 102666.672319, "command_buffers": 2 }, { "command_buffer_ns": 20041.6543521, "command_buffers": 2 }, { "command_buffer_ns": 127041.654196, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 19.79101489265625, 16.1881507665625, 22.254557734375002, 16.52929586265625, 23.409504137812498, 16.5501296578125, 20.777994905, 16.14648408578125, 30.5345047309375 ], "command_buffer_throughput_us_p50": 19.79101489265625, "command_buffer_throughput_us_p90": 24.834504256437498, "command_buffer_latency_us": [ 65.0000292808, 19.999977666899998, 94.7500229813, 20.125007722499998, 84.4582682475, 20.2499795705, 102.666672319, 20.0416543521, 127.041654196 ], "command_buffer_latency_us_p50": 65.0000292808, "command_buffer_latency_us_p90": 107.5416686944 }, "compute_throughput_us": [ 16.220703125, 23.353515625, 16.66015625, 23.55078125, 16.146484375, 23.125, 16.317703125, 17.298171875, 16.046875 ], "compute_throughput_us_p50": 16.66015625, "compute_throughput_us_p90": 23.39296875, "command_buffer_throughput_us": [ 16.2220048878125, 23.35351564390625, 16.6621093740625, 23.551432605000002, 16.14778557265625, 23.12695414734375, 16.318359485000002, 17.29947962296875, 16.0475265146875 ], "command_buffer_throughput_us_p50": 16.6621093740625, "command_buffer_throughput_us_p90": 23.393099036125, "compute_latency_us": [ 22.583, 17.959, 22.25, 17.958, 20.291, 18.083, 21.791, 18.5, 22.375 ], "compute_latency_us_p50": 20.291, "compute_latency_us_p90": 22.4166, "command_buffer_latency_us": [ 22.7083801292, 18.0416973308, 22.2916714847, 17.958343960300002, 20.3332747333, 18.208287656299998, 21.874962840200002, 18.541701138, 22.4582618102 ], "command_buffer_latency_us_p50": 20.3332747333, "command_buffer_latency_us_p90": 22.508285474, "counter_control_throughput_ratio": 0.8196651346986715, "counter_control_latency_ratio": 0.34935953691805033 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 23.4223098495, "throughput_us_p90": 24.08495800278, "latency_us_p50": 250.833, "latency_us_p90": 710.0082000000004 }, "slowdown": 0.3839215829288768, "device_compute_slowdown": 0.40026277509267516, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-c0s8ojah/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "4f0c4e70dcb06b507aa82e242c8819acd3628d5f79a065eb6c2f6b73ec6f396c", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.353834, "capture_ms": 0.071, "compile_ms": 64.197666, "allocation_upload_ms": 7.027417, "cold_call_ms": 5.986875, "warmup_ms": 201.866666, "download_ms": 2.603208, "repetitions": 253, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 106.361166008, 113.195652174, 107.886857708, 106.551715415, 115.991928854, 105.709814229, 106.336790514, 116.698288538, 106.248683794 ], "latency_us": [ 335.042, 328.208, 330.25, 372.625, 326.166, 296.042, 317.208, 2065.083, 949.083 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 5153333, "compute_span_ns": 5153333, "calibration_cpu_ns": 6139333, "calibration_gpu_ticks": 6139333, "compute_passes": 1, "command_buffer_ns": 5153416.65596, "command_buffers": 2 }, { "compute_ns": 5079334, "compute_span_ns": 5079334, "calibration_cpu_ns": 6534958, "calibration_gpu_ticks": 6534958, "compute_passes": 1, "command_buffer_ns": 5079458.35125, "command_buffers": 2 }, { "compute_ns": 5138083, "compute_span_ns": 5138083, "calibration_cpu_ns": 6454167, "calibration_gpu_ticks": 6454167, "compute_passes": 1, "command_buffer_ns": 5138166.71446, "command_buffers": 2 }, { "compute_ns": 5025375, "compute_span_ns": 5025375, "calibration_cpu_ns": 6237750, "calibration_gpu_ticks": 6237750, "compute_passes": 1, "command_buffer_ns": 5025499.966, "command_buffers": 2 }, { "compute_ns": 4963208, "compute_span_ns": 4963208, "calibration_cpu_ns": 6534666, "calibration_gpu_ticks": 6534666, "compute_passes": 1, "command_buffer_ns": 4963208.28097, "command_buffers": 2 }, { "compute_ns": 5174167, "compute_span_ns": 5174167, "calibration_cpu_ns": 6250750, "calibration_gpu_ticks": 6250750, "compute_passes": 1, "command_buffer_ns": 5174249.99271, "command_buffers": 2 }, { "compute_ns": 5078416, "compute_span_ns": 5078416, "calibration_cpu_ns": 6469042, "calibration_gpu_ticks": 6469042, "compute_passes": 1, "command_buffer_ns": 5078458.34363, "command_buffers": 2 }, { "compute_ns": 7510542, "compute_span_ns": 7510542, "calibration_cpu_ns": 8841583, "calibration_gpu_ticks": 8841583, "compute_passes": 1, "command_buffer_ns": 7510625.00058, "command_buffers": 2 }, { "compute_ns": 7390541, "compute_span_ns": 7390541, "calibration_cpu_ns": 8586250, "calibration_gpu_ticks": 8586250, "compute_passes": 1, "command_buffer_ns": 7390541.723, "command_buffers": 2 } ], "latency": [ { "compute_ns": 104791, "compute_span_ns": 104791, "calibration_cpu_ns": 696416, "calibration_gpu_ticks": 696416, "compute_passes": 1, "command_buffer_ns": 104875.070974, "command_buffers": 2 }, { "compute_ns": 76333, "compute_span_ns": 76333, "calibration_cpu_ns": 584750, "calibration_gpu_ticks": 584750, "compute_passes": 1, "command_buffer_ns": 76416.7052694, "command_buffers": 2 }, { "compute_ns": 118125, "compute_span_ns": 118125, "calibration_cpu_ns": 1143459, "calibration_gpu_ticks": 1143459, "compute_passes": 1, "command_buffer_ns": 118166.732136, "command_buffers": 2 }, { "compute_ns": 119375, "compute_span_ns": 119375, "calibration_cpu_ns": 3082041, "calibration_gpu_ticks": 3082041, "compute_passes": 1, "command_buffer_ns": 119458.360132, "command_buffers": 2 }, { "compute_ns": 80291, "compute_span_ns": 80291, "calibration_cpu_ns": 523375, "calibration_gpu_ticks": 523375, "compute_passes": 1, "command_buffer_ns": 80250.0289865, "command_buffers": 2 }, { "compute_ns": 81625, "compute_span_ns": 81625, "calibration_cpu_ns": 635833, "calibration_gpu_ticks": 635833, "compute_passes": 1, "command_buffer_ns": 81666.6288301, "command_buffers": 2 }, { "compute_ns": 75875, "compute_span_ns": 75875, "calibration_cpu_ns": 635667, "calibration_gpu_ticks": 635667, "compute_passes": 1, "command_buffer_ns": 75916.6432545, "command_buffers": 2 }, { "compute_ns": 120458, "compute_span_ns": 120458, "calibration_cpu_ns": 2035250, "calibration_gpu_ticks": 2035250, "compute_passes": 1, "command_buffer_ns": 120541.604701, "command_buffers": 2 }, { "compute_ns": 114083, "compute_span_ns": 114083, "calibration_cpu_ns": 3121208, "calibration_gpu_ticks": 3121208, "compute_passes": 1, "command_buffer_ns": 114083.348308, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5112625.01823, "command_buffers": 2 }, { "command_buffer_ns": 5311999.99386, "command_buffers": 2 }, { "command_buffer_ns": 4897291.59744, "command_buffers": 2 }, { "command_buffer_ns": 5273333.285, "command_buffers": 2 }, { "command_buffer_ns": 5319208.37238, "command_buffers": 2 }, { "command_buffer_ns": 5231250.01928, "command_buffers": 2 }, { "command_buffer_ns": 5097791.66896, "command_buffers": 2 }, { "command_buffer_ns": 5321083.35756, "command_buffers": 2 }, { "command_buffer_ns": 4874041.59503, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 226375.006605, "command_buffers": 2 }, { "command_buffer_ns": 74624.9570511, "command_buffers": 2 }, { "command_buffer_ns": 120125.012472, "command_buffers": 2 }, { "command_buffer_ns": 75541.6695029, "command_buffers": 2 }, { "command_buffer_ns": 118708.354421, "command_buffers": 2 }, { "command_buffer_ns": 149916.682858, "command_buffers": 2 }, { "command_buffer_ns": 99166.7038761, "command_buffers": 2 }, { "command_buffer_ns": 76416.7052694, "command_buffers": 2 }, { "command_buffer_ns": 112625.013571, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 79.88476590984375, 82.99999990406249, 76.52018120999999, 82.395832578125, 83.1126308184375, 81.73828155125, 79.65299482750001, 83.141927461875, 76.15689992234375 ], "command_buffer_throughput_us_p50": 81.73828155125, "command_buffer_throughput_us_p90": 83.118490147125, "command_buffer_latency_us": [ 226.375006605, 74.62495705110001, 120.12501247200001, 75.54166950289999, 118.708354421, 149.91668285799997, 99.1667038761, 76.4167052694, 112.625013571 ], "command_buffer_latency_us_p50": 112.625013571, "command_buffer_latency_us_p90": 165.2083476074 }, "compute_throughput_us": [ 80.520828125, 79.36459375, 80.282546875, 78.521484375, 77.550125, 80.846359375, 79.35025, 117.35221875, 115.477203125 ], "compute_throughput_us_p50": 80.282546875, "compute_throughput_us_p90": 115.85220625000001, "command_buffer_throughput_us": [ 80.522135249375, 79.36653673828125, 80.28385491343751, 78.52343696875, 77.55012939015624, 80.84765613609375, 79.35091161921875, 117.35351563406249, 115.477214421875 ], "command_buffer_throughput_us_p50": 80.28385491343751, "command_buffer_throughput_us_p90": 115.8524746643125, "compute_latency_us": [ 104.791, 76.333, 118.125, 119.375, 80.291, 81.625, 75.875, 120.458, 114.083 ], "compute_latency_us_p50": 104.791, "compute_latency_us_p90": 119.5916, "command_buffer_latency_us": [ 104.875070974, 76.4167052694, 118.16673213600001, 119.458360132, 80.2500289865, 81.6666288301, 75.9166432545, 120.541604701, 114.083348308 ], "command_buffer_latency_us_p50": 104.875070974, "command_buffer_latency_us_p90": 119.6750090458, "counter_control_throughput_ratio": 0.9962075097247071, "counter_control_latency_ratio": 0.9836979801649848 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 106.551715415, "throughput_us_p90": 116.13320079079999, "latency_us_p50": 330.25, "latency_us_p90": 1172.2830000000001 }, "torch": { "cold_call_ms": 8.439333, "warmup_ms": 202.5675, "repetitions": 77, "throughput_us": [ 374.012987012987, 412.92802597402596, 402.9778181818182, 373.82305194805195, 412.93776623376624, 406.0021558441559, 371.4529220779221, 408.0470779220779, 406.5173116883117 ], "latency_us": [ 1025.917, 517.458, 504.54200000000003, 502.12500000000006, 510.95900000000006, 504.75000000000006, 504.708, 2737.917, 828.917 ], "throughput_us_p50": 406.0021558441559, "throughput_us_p90": 412.929974025974, "latency_us_p50": 510.95900000000006, "latency_us_p90": 1368.3170000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 24847624.998074025, "command_buffers": 1 }, { "command_buffer_ns": 22655749.98455122, "command_buffers": 1 }, { "command_buffer_ns": 22365499.986335635, "command_buffers": 1 }, { "command_buffer_ns": 21900416.642893106, "command_buffers": 1 }, { "command_buffer_ns": 23084166.692569852, "command_buffers": 1 }, { "command_buffer_ns": 22942291.689105332, "command_buffers": 1 }, { "command_buffer_ns": 22412833.35024491, "command_buffers": 1 }, { "command_buffer_ns": 22184874.9788478, "command_buffers": 1 }, { "command_buffer_ns": 22180625.004693866, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 278250.02325698733, "command_buffers": 1 }, { "command_buffer_ns": 579125.015065074, "command_buffers": 1 }, { "command_buffer_ns": 277208.3389572799, "command_buffers": 1 }, { "command_buffer_ns": 287749.9791793525, "command_buffers": 1 }, { "command_buffer_ns": 285250.01835078, "command_buffers": 1 }, { "command_buffer_ns": 284375.04079192877, "command_buffers": 1 }, { "command_buffer_ns": 271625.0019147992, "command_buffers": 1 }, { "command_buffer_ns": 286124.99590963125, "command_buffers": 1 }, { "command_buffer_ns": 271375.00001117587, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 388.24414059490664, 353.99609350861283, 349.4609372864943, 342.1940100452048, 360.69010457140394, 358.4733076422708, 350.2005210975767, 346.6386715444969, 346.57226569834165 ], "command_buffer_throughput_us_p50": 350.2005210975767, "command_buffer_throughput_us_p90": 366.2009117761045, "command_buffer_latency_us": [ 278.25002325698733, 579.125015065074, 277.2083389572799, 287.7499791793525, 285.25001835078, 284.37504079192877, 271.6250019147992, 286.12499590963125, 271.37500001117587 ], "command_buffer_latency_us_p50": 284.37504079192877, "command_buffer_latency_us_p90": 346.02498635649687 }, "throughput": [ { "compute_ns": 22779250.0, "compute_span_ns": 22779250.0, "command_buffer_ns": 22779249.993618578, "calibration_cpu_ns": 25334625.0, "calibration_gpu_ticks": 25334625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22036333.0, "compute_span_ns": 22036333.0, "command_buffer_ns": 22036291.658878326, "calibration_cpu_ns": 24205958.0, "calibration_gpu_ticks": 24205958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21867792.0, "compute_span_ns": 21867792.0, "command_buffer_ns": 21867833.333089948, "calibration_cpu_ns": 23042833.0, "calibration_gpu_ticks": 23042833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22007417.0, "compute_span_ns": 22007417.0, "command_buffer_ns": 22007458.37762952, "calibration_cpu_ns": 23874083.0, "calibration_gpu_ticks": 23874083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22046417.0, "compute_span_ns": 22046417.0, "command_buffer_ns": 22046416.70687124, "calibration_cpu_ns": 23586041.0, "calibration_gpu_ticks": 23586041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22156125.0, "compute_span_ns": 22156125.0, "command_buffer_ns": 22156124.99276176, "calibration_cpu_ns": 24132209.0, "calibration_gpu_ticks": 24132209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21994334.0, "compute_span_ns": 21994334.0, "command_buffer_ns": 21994374.983478338, "calibration_cpu_ns": 24122375.0, "calibration_gpu_ticks": 24122375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 25117583.0, "compute_span_ns": 25117583.0, "command_buffer_ns": 25117583.34003389, "calibration_cpu_ns": 28160792.0, "calibration_gpu_ticks": 28160792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21868250.0, "compute_span_ns": 21868250.0, "command_buffer_ns": 21868249.983526766, "calibration_cpu_ns": 23696166.0, "calibration_gpu_ticks": 23696166.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 271875.0, "compute_span_ns": 271875.0, "command_buffer_ns": 271875.00381842256, "calibration_cpu_ns": 1060291.0, "calibration_gpu_ticks": 1060291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 285416.0, "compute_span_ns": 285416.0, "command_buffer_ns": 285374.9901987612, "calibration_cpu_ns": 1599250.0, "calibration_gpu_ticks": 1599250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275625.0, "compute_span_ns": 275625.0, "command_buffer_ns": 275625.0323727727, "calibration_cpu_ns": 761250.0, "calibration_gpu_ticks": 761250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 273208.0, "compute_span_ns": 273208.0, "command_buffer_ns": 273166.6318140924, "calibration_cpu_ns": 1179583.0, "calibration_gpu_ticks": 1179583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 304542.0, "compute_span_ns": 304542.0, "command_buffer_ns": 304583.3436772227, "calibration_cpu_ns": 1366042.0, "calibration_gpu_ticks": 1366042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271083.0, "compute_span_ns": 271083.0, "command_buffer_ns": 271083.3214223385, "calibration_cpu_ns": 718291.0, "calibration_gpu_ticks": 718291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271375.0, "compute_span_ns": 271375.0, "command_buffer_ns": 271375.00001117587, "calibration_cpu_ns": 730542.0, "calibration_gpu_ticks": 730542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 287917.0, "compute_span_ns": 287917.0, "command_buffer_ns": 287916.6277125478, "calibration_cpu_ns": 1512750.0, "calibration_gpu_ticks": 1512750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 284083.0, "compute_span_ns": 284083.0, "command_buffer_ns": 284083.30399543047, "calibration_cpu_ns": 3400458.0, "calibration_gpu_ticks": 3400458.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 355.92578125, 344.317703125, 341.68425, 343.865890625, 344.475265625, 346.189453125, 343.66146875, 392.462234375, 341.69140625 ], "compute_throughput_us_p50": 344.317703125, "compute_throughput_us_p90": 363.233071875, "command_buffer_throughput_us": [ 355.9257811502903, 344.31705716997385, 341.68489582953043, 343.8665371504612, 344.47526104486315, 346.1894530119025, 343.662109116849, 392.4622396880295, 341.6914059926057 ], "command_buffer_throughput_us_p50": 344.31705716997385, "command_buffer_throughput_us_p90": 363.2330728578381, "compute_latency_us": [ 271.875, 285.416, 275.625, 273.208, 304.542, 271.083, 271.375, 287.917, 284.083 ], "compute_latency_us_p50": 275.625, "compute_latency_us_p90": 291.24199999999996, "command_buffer_latency_us": [ 271.87500381842256, 285.3749901987612, 275.6250323727727, 273.1666318140924, 304.58334367722273, 271.0833214223385, 271.37500001117587, 287.9166277125478, 284.08330399543047 ], "command_buffer_latency_us_p50": 275.6250323727727, "command_buffer_latency_us_p90": 291.24997090548277, "counter_control_throughput_ratio": 0.9777484673470416, "counter_control_latency_ratio": 0.9942883876060048 }, "download_ms": 0.774583, "allocation_upload_ms": 20.225666, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.2624412552525951, "device_compute_slowdown": 0.2331641566679901, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 1.129, "warmup_ms": 202.836583, "repetitions": 80, "throughput_us": [ 393.5421875, 375.245825, 386.39480000000003, 398.1515625, 383.9828125, 370.0448, 400.9416625, 402.75103750000005, 383.19635000000005 ], "latency_us": [ 1115.583, 787.875, 479.08299999999997, 524.6659999999999, 529.792, 513.0, 513.417, 510.375, 512.3330000000001 ], "throughput_us_p50": 386.39480000000003, "throughput_us_p90": 401.3035375, "latency_us_p50": 513.417, "latency_us_p90": 853.4166, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 22108124.976512045, "command_buffers": 1 }, { "command_buffer_ns": 22197916.638106108, "command_buffers": 1 }, { "command_buffer_ns": 22184958.33221823, "command_buffers": 1 }, { "command_buffer_ns": 23951500.01393631, "command_buffers": 1 }, { "command_buffer_ns": 24313000.0308156, "command_buffers": 1 }, { "command_buffer_ns": 22383791.68557003, "command_buffers": 1 }, { "command_buffer_ns": 22074625.01246482, "command_buffers": 1 }, { "command_buffer_ns": 22569749.96998906, "command_buffers": 1 }, { "command_buffer_ns": 22499000.013340265, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 309249.96826797724, "command_buffers": 1 }, { "command_buffer_ns": 274999.9985098839, "command_buffers": 1 }, { "command_buffer_ns": 271083.3796299994, "command_buffers": 1 }, { "command_buffer_ns": 271291.7048484087, "command_buffers": 1 }, { "command_buffer_ns": 514583.31290632486, "command_buffers": 1 }, { "command_buffer_ns": 454624.998383224, "command_buffers": 1 }, { "command_buffer_ns": 276624.98177960515, "command_buffers": 1 }, { "command_buffer_ns": 280208.36180076003, "command_buffers": 1 }, { "command_buffer_ns": 270583.3758227527, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 345.4394527580007, 346.84244747040793, 346.63997394090984, 374.24218771775486, 379.89062548149377, 349.74674508703174, 344.91601581976283, 352.6523432810791, 351.54687520844163 ], "command_buffer_throughput_us_p50": 349.74674508703174, "command_buffer_throughput_us_p90": 375.37187527050264, "command_buffer_latency_us": [ 309.24996826797724, 274.9999985098839, 271.0833796299994, 271.2917048484087, 514.5833129063249, 454.624998383224, 276.62498177960515, 280.20836180076003, 270.5833758227527 ], "command_buffer_latency_us_p50": 276.62498177960515, "command_buffer_latency_us_p90": 466.6166612878442 }, "throughput": [ { "compute_ns": 22723083.0, "compute_span_ns": 22723083.0, "command_buffer_ns": 22723083.32612738, "calibration_cpu_ns": 24814500.0, "calibration_gpu_ticks": 24814500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22308167.0, "compute_span_ns": 22308167.0, "command_buffer_ns": 22308208.339381963, "calibration_cpu_ns": 24842625.0, "calibration_gpu_ticks": 24842625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21870125.0, "compute_span_ns": 21870125.0, "command_buffer_ns": 21870124.96870011, "calibration_cpu_ns": 23445666.0, "calibration_gpu_ticks": 23445666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21860375.0, "compute_span_ns": 21860375.0, "command_buffer_ns": 21860375.010874122, "calibration_cpu_ns": 24474708.0, "calibration_gpu_ticks": 24474708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 25396459.0, "compute_span_ns": 25396459.0, "command_buffer_ns": 25396500.015631318, "calibration_cpu_ns": 26543458.0, "calibration_gpu_ticks": 26543458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21933083.0, "compute_span_ns": 21933083.0, "command_buffer_ns": 21933041.68758914, "calibration_cpu_ns": 24401334.0, "calibration_gpu_ticks": 24401334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21951458.0, "compute_span_ns": 21951458.0, "command_buffer_ns": 21951416.681986302, "calibration_cpu_ns": 24474875.0, "calibration_gpu_ticks": 24474875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22666667.0, "compute_span_ns": 22666667.0, "command_buffer_ns": 22666708.333417773, "calibration_cpu_ns": 24946000.0, "calibration_gpu_ticks": 24946000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22548208.0, "compute_span_ns": 22548208.0, "command_buffer_ns": 22548208.362422884, "calibration_cpu_ns": 24131584.0, "calibration_gpu_ticks": 24131584.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 271417.0, "compute_span_ns": 271417.0, "command_buffer_ns": 271416.6766963899, "calibration_cpu_ns": 744042.0, "calibration_gpu_ticks": 744042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 382333.0, "compute_span_ns": 382333.0, "command_buffer_ns": 382291.6769422591, "calibration_cpu_ns": 1988750.0, "calibration_gpu_ticks": 1988750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 595625.0, "compute_span_ns": 595625.0, "command_buffer_ns": 595624.9660812318, "calibration_cpu_ns": 1600167.0, "calibration_gpu_ticks": 1600167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 277084.0, "compute_span_ns": 277084.0, "command_buffer_ns": 277125.04379451275, "calibration_cpu_ns": 825667.0, "calibration_gpu_ticks": 825667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 278958.0, "compute_span_ns": 278958.0, "command_buffer_ns": 278916.6755974293, "calibration_cpu_ns": 989292.0, "calibration_gpu_ticks": 989292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275834.0, "compute_span_ns": 275834.0, "command_buffer_ns": 275874.9760687351, "calibration_cpu_ns": 733417.0, "calibration_gpu_ticks": 733417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270375.0, "compute_span_ns": 270375.0, "command_buffer_ns": 270375.0506043434, "calibration_cpu_ns": 714375.0, "calibration_gpu_ticks": 714375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 533375.0, "compute_span_ns": 533375.0, "command_buffer_ns": 533375.0159479678, "calibration_cpu_ns": 2089000.0, "calibration_gpu_ticks": 2089000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 564792.0, "compute_span_ns": 564792.0, "command_buffer_ns": 564833.3462886512, "calibration_cpu_ns": 1912959.0, "calibration_gpu_ticks": 1912959.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 355.048171875, 348.565109375, 341.720703125, 341.568359375, 396.819671875, 342.704421875, 342.99153125, 354.166671875, 352.31575 ], "compute_throughput_us_p50": 348.565109375, "compute_throughput_us_p90": 363.402471875, "command_buffer_throughput_us": [ 355.0481769707403, 348.5657553028432, 341.72070263593923, 341.56835954490816, 396.82031274423935, 342.7037763685803, 342.99088565603597, 354.1673177096527, 352.31575566285755 ], "command_buffer_throughput_us_p50": 348.5657553028432, "command_buffer_throughput_us_p90": 363.4026041254401, "compute_latency_us": [ 271.417, 382.333, 595.625, 277.084, 278.958, 275.834, 270.375, 533.375, 564.792 ], "compute_latency_us_p50": 278.958, "compute_latency_us_p90": 570.9586, "command_buffer_latency_us": [ 271.4166766963899, 382.2916769422591, 595.6249660812318, 277.12504379451275, 278.9166755974293, 275.8749760687351, 270.3750506043434, 533.3750159479678, 564.8333462886512 ], "command_buffer_latency_us_p50": 278.9166755974293, "command_buffer_latency_us_p90": 570.9916702471673, "counter_control_throughput_ratio": 1.002187134941706, "counter_control_latency_ratio": 1.0215020910770698 }, "download_ms": 0.918167, "allocation_upload_ms": 1.164791, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-2_m5sb4g/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "2f415c8199d32d1b999ce6dae6674c9bc8bbdf9b0adda641a6d2aee649468028", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.699959, "capture_ms": 0.085375, "compile_ms": 65.99675, "allocation_upload_ms": 8.148, "cold_call_ms": 8.441167, "warmup_ms": 200.409041, "download_ms": 3.406916, "repetitions": 287, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 105.319250871, 103.50087108, 109.627756098, 102.355111498, 107.041376307, 110.604965157, 102.472996516, 107.923923345, 110.986062718 ], "latency_us": [ 346.708, 317.375, 296.375, 321.875, 274.625, 298.333, 2543.958, 648.375, 569.25 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4587083, "compute_span_ns": 4587083, "calibration_cpu_ns": 6617958, "calibration_gpu_ticks": 6617958, "compute_passes": 1, "command_buffer_ns": 4587166.70983, "command_buffers": 2 }, { "compute_ns": 4721875, "compute_span_ns": 4721875, "calibration_cpu_ns": 6322958, "calibration_gpu_ticks": 6322958, "compute_passes": 1, "command_buffer_ns": 4721999.9833, "command_buffers": 2 }, { "compute_ns": 5300791, "compute_span_ns": 5300791, "calibration_cpu_ns": 6377792, "calibration_gpu_ticks": 6377792, "compute_passes": 1, "command_buffer_ns": 5300875.05467, "command_buffers": 2 }, { "compute_ns": 5017583, "compute_span_ns": 5017583, "calibration_cpu_ns": 6152084, "calibration_gpu_ticks": 6152084, "compute_passes": 1, "command_buffer_ns": 5017541.63997, "command_buffers": 2 }, { "compute_ns": 5072500, "compute_span_ns": 5072500, "calibration_cpu_ns": 6262875, "calibration_gpu_ticks": 6262875, "compute_passes": 1, "command_buffer_ns": 5072625.00469, "command_buffers": 2 }, { "compute_ns": 4857375, "compute_span_ns": 4857375, "calibration_cpu_ns": 6206417, "calibration_gpu_ticks": 6206417, "compute_passes": 1, "command_buffer_ns": 4857499.96733, "command_buffers": 2 }, { "compute_ns": 4935208, "compute_span_ns": 4935208, "calibration_cpu_ns": 6205542, "calibration_gpu_ticks": 6205542, "compute_passes": 1, "command_buffer_ns": 4935333.27244, "command_buffers": 2 }, { "compute_ns": 4853000, "compute_span_ns": 4853000, "calibration_cpu_ns": 6278834, "calibration_gpu_ticks": 6278834, "compute_passes": 1, "command_buffer_ns": 4853083.34464, "command_buffers": 2 }, { "compute_ns": 7296959, "compute_span_ns": 7296959, "calibration_cpu_ns": 8298250, "calibration_gpu_ticks": 8298250, "compute_passes": 1, "command_buffer_ns": 7297041.70953, "command_buffers": 2 } ], "latency": [ { "compute_ns": 118958, "compute_span_ns": 118958, "calibration_cpu_ns": 3865708, "calibration_gpu_ticks": 3865708, "compute_passes": 1, "command_buffer_ns": 118999.974802, "command_buffers": 2 }, { "compute_ns": 124042, "compute_span_ns": 124042, "calibration_cpu_ns": 1354041, "calibration_gpu_ticks": 1354041, "compute_passes": 1, "command_buffer_ns": 124083.308037, "command_buffers": 2 }, { "compute_ns": 121458, "compute_span_ns": 121458, "calibration_cpu_ns": 1214125, "calibration_gpu_ticks": 1214125, "compute_passes": 1, "command_buffer_ns": 121541.670524, "command_buffers": 2 }, { "compute_ns": 118458, "compute_span_ns": 118458, "calibration_cpu_ns": 1207417, "calibration_gpu_ticks": 1207417, "compute_passes": 1, "command_buffer_ns": 118541.705888, "command_buffers": 2 }, { "compute_ns": 95916, "compute_span_ns": 95916, "calibration_cpu_ns": 667708, "calibration_gpu_ticks": 667708, "compute_passes": 1, "command_buffer_ns": 95916.679129, "command_buffers": 2 }, { "compute_ns": 124583, "compute_span_ns": 124583, "calibration_cpu_ns": 2880542, "calibration_gpu_ticks": 2880542, "compute_passes": 1, "command_buffer_ns": 124583.311845, "command_buffers": 2 }, { "compute_ns": 125000, "compute_span_ns": 125000, "calibration_cpu_ns": 1461833, "calibration_gpu_ticks": 1461833, "compute_passes": 1, "command_buffer_ns": 125000.020489, "command_buffers": 2 }, { "compute_ns": 73541, "compute_span_ns": 73541, "calibration_cpu_ns": 531041, "calibration_gpu_ticks": 531041, "compute_passes": 1, "command_buffer_ns": 73500.0357963, "command_buffers": 2 }, { "compute_ns": 121959, "compute_span_ns": 121959, "calibration_cpu_ns": 1505417, "calibration_gpu_ticks": 1505417, "compute_passes": 1, "command_buffer_ns": 122041.674331, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5538875.06248, "command_buffers": 2 }, { "command_buffer_ns": 5134666.6296, "command_buffers": 2 }, { "command_buffer_ns": 5096458.30607, "command_buffers": 2 }, { "command_buffer_ns": 5021458.31706, "command_buffers": 2 }, { "command_buffer_ns": 4937708.37784, "command_buffers": 2 }, { "command_buffer_ns": 5048833.32178, "command_buffers": 2 }, { "command_buffer_ns": 4816416.70922, "command_buffers": 2 }, { "command_buffer_ns": 7481583.39411, "command_buffers": 2 }, { "command_buffer_ns": 7273083.31989, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 118291.703984, "command_buffers": 2 }, { "command_buffer_ns": 121375.02199, "command_buffers": 2 }, { "command_buffer_ns": 73583.3309591, "command_buffers": 2 }, { "command_buffer_ns": 91083.3477974, "command_buffers": 2 }, { "command_buffer_ns": 123916.659504, "command_buffers": 2 }, { "command_buffer_ns": 73500.0357963, "command_buffers": 2 }, { "command_buffer_ns": 73166.6223146, "command_buffers": 2 }, { "command_buffer_ns": 125041.638967, "command_buffers": 2 }, { "command_buffer_ns": 73125.003837, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 86.54492285124999, 80.22916608749999, 79.63216103234375, 78.4602862040625, 77.15169340375, 78.8880206528125, 75.2565110815625, 116.89974053296875, 113.64192687328125 ], "command_buffer_throughput_us_p50": 79.63216103234375, "command_buffer_throughput_us_p90": 114.29348960521875, "command_buffer_latency_us": [ 118.29170398400001, 121.37502199, 73.5833309591, 91.0833477974, 123.916659504, 73.5000357963, 73.16662231459999, 125.04163896700001, 73.125003837 ], "command_buffer_latency_us_p50": 91.0833477974, "command_buffer_latency_us_p90": 124.1416553966 }, "compute_throughput_us": [ 71.673171875, 73.779296875, 82.824859375, 78.399734375, 79.2578125, 75.896484375, 77.112625, 75.828125, 114.014984375 ], "compute_throughput_us_p50": 77.112625, "compute_throughput_us_p90": 89.06288437500001, "command_buffer_throughput_us": [ 71.67447984109376, 73.7812497390625, 82.82617272921875, 78.39908812453125, 79.25976569828124, 75.89843698953126, 77.11458238187501, 75.82942726, 114.01627671140625 ], "command_buffer_throughput_us_p50": 77.11458238187501, "command_buffer_throughput_us_p90": 89.06419352565626, "compute_latency_us": [ 118.958, 124.042, 121.458, 118.458, 95.916, 124.583, 125.0, 73.541, 121.959 ], "compute_latency_us_p50": 121.458, "compute_latency_us_p90": 124.6664, "command_buffer_latency_us": [ 118.999974802, 124.083308037, 121.541670524, 118.541705888, 95.916679129, 124.583311845, 125.00002048900001, 73.5000357963, 122.041674331 ], "command_buffer_latency_us_p50": 121.541670524, "command_buffer_latency_us_p90": 124.6666535738, "counter_control_throughput_ratio": 0.9992200120278418, "counter_control_latency_ratio": 1.3014640848695704 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 107.041376307, "throughput_us_p90": 110.6811846692, "latency_us_p50": 321.875, "latency_us_p90": 1027.4916000000003 }, "slowdown": 0.27702592350362887, "device_compute_slowdown": 0.22122875447364185, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-mab3qp4p/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7d02ca31ce0660bb7b8cf2d4b7c68e2de8afe58cfb625e16c6aa93173ce8f234", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.664458, "capture_ms": 0.087708, "compile_ms": 64.953208, "allocation_upload_ms": 6.981958, "cold_call_ms": 9.456166, "warmup_ms": 200.152, "download_ms": 3.410166, "repetitions": 285, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 111.756726316, 105.60877193, 110.462719298, 113.051315789, 112.914768421, 112.288157895, 108.031873684, 107.056578947, 116.772512281 ], "latency_us": [ 359.25, 296.167, 610.916, 2236.584, 426.459, 530.834, 304.042, 286.041, 365 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4993500, "compute_span_ns": 4993500, "calibration_cpu_ns": 6317542, "calibration_gpu_ticks": 6317542, "compute_passes": 1, "command_buffer_ns": 4993708.33859, "command_buffers": 2 }, { "compute_ns": 5120208, "compute_span_ns": 5120208, "calibration_cpu_ns": 6400875, "calibration_gpu_ticks": 6400875, "compute_passes": 1, "command_buffer_ns": 5120250.04718, "command_buffers": 2 }, { "compute_ns": 5117625, "compute_span_ns": 5117625, "calibration_cpu_ns": 6023417, "calibration_gpu_ticks": 6023417, "compute_passes": 1, "command_buffer_ns": 5117708.35146, "command_buffers": 2 }, { "compute_ns": 5093833, "compute_span_ns": 5093833, "calibration_cpu_ns": 6262167, "calibration_gpu_ticks": 6262167, "compute_passes": 1, "command_buffer_ns": 5093874.93366, "command_buffers": 2 }, { "compute_ns": 5278000, "compute_span_ns": 5278000, "calibration_cpu_ns": 6513208, "calibration_gpu_ticks": 6513208, "compute_passes": 1, "command_buffer_ns": 5278041.64449, "command_buffers": 2 }, { "compute_ns": 4946750, "compute_span_ns": 4946750, "calibration_cpu_ns": 6452708, "calibration_gpu_ticks": 6452708, "compute_passes": 1, "command_buffer_ns": 4946791.68332, "command_buffers": 2 }, { "compute_ns": 5110833, "compute_span_ns": 5110833, "calibration_cpu_ns": 6206292, "calibration_gpu_ticks": 6206292, "compute_passes": 1, "command_buffer_ns": 5110958.30007, "command_buffers": 2 }, { "compute_ns": 5240250, "compute_span_ns": 5240250, "calibration_cpu_ns": 6502667, "calibration_gpu_ticks": 6502667, "compute_passes": 1, "command_buffer_ns": 5240333.32476, "command_buffers": 2 }, { "compute_ns": 7375917, "compute_span_ns": 7375917, "calibration_cpu_ns": 8565334, "calibration_gpu_ticks": 8565334, "compute_passes": 1, "command_buffer_ns": 7375958.37563, "command_buffers": 2 } ], "latency": [ { "compute_ns": 113166, "compute_span_ns": 113166, "calibration_cpu_ns": 1360042, "calibration_gpu_ticks": 1360042, "compute_passes": 1, "command_buffer_ns": 113208.312541, "command_buffers": 2 }, { "compute_ns": 116166, "compute_span_ns": 116166, "calibration_cpu_ns": 1968333, "calibration_gpu_ticks": 1968333, "compute_passes": 1, "command_buffer_ns": 116124.982014, "command_buffers": 2 }, { "compute_ns": 76792, "compute_span_ns": 76792, "calibration_cpu_ns": 487333, "calibration_gpu_ticks": 487333, "compute_passes": 1, "command_buffer_ns": 76833.2974985, "command_buffers": 2 }, { "compute_ns": 343250, "compute_span_ns": 343250, "calibration_cpu_ns": 1766292, "calibration_gpu_ticks": 1766292, "compute_passes": 1, "command_buffer_ns": 343333.347701, "command_buffers": 2 }, { "compute_ns": 77125, "compute_span_ns": 77125, "calibration_cpu_ns": 530500, "calibration_gpu_ticks": 530500, "compute_passes": 1, "command_buffer_ns": 77166.6527726, "command_buffers": 2 }, { "compute_ns": 111500, "compute_span_ns": 111500, "calibration_cpu_ns": 736584, "calibration_gpu_ticks": 736584, "compute_passes": 1, "command_buffer_ns": 111583.329272, "command_buffers": 2 }, { "compute_ns": 75959, "compute_span_ns": 75959, "calibration_cpu_ns": 554875, "calibration_gpu_ticks": 554875, "compute_passes": 1, "command_buffer_ns": 76041.6733101, "command_buffers": 2 }, { "compute_ns": 77416, "compute_span_ns": 77416, "calibration_cpu_ns": 535708, "calibration_gpu_ticks": 535708, "compute_passes": 1, "command_buffer_ns": 77374.977991, "command_buffers": 2 }, { "compute_ns": 76250, "compute_span_ns": 76250, "calibration_cpu_ns": 523458, "calibration_gpu_ticks": 523458, "compute_passes": 1, "command_buffer_ns": 76333.2936913, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5149583.33224, "command_buffers": 2 }, { "command_buffer_ns": 5306708.33541, "command_buffers": 2 }, { "command_buffer_ns": 5217000.02719, "command_buffers": 2 }, { "command_buffer_ns": 5100624.98506, "command_buffers": 2 }, { "command_buffer_ns": 5269958.34662, "command_buffers": 2 }, { "command_buffer_ns": 5299166.60162, "command_buffers": 2 }, { "command_buffer_ns": 5267458.26937, "command_buffers": 2 }, { "command_buffer_ns": 7649999.98501, "command_buffers": 2 }, { "command_buffer_ns": 7577624.98688, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 113458.256237, "command_buffers": 2 }, { "command_buffer_ns": 75999.9966249, "command_buffers": 2 }, { "command_buffer_ns": 76375.0285842, "command_buffers": 2 }, { "command_buffer_ns": 106916.588265, "command_buffers": 2 }, { "command_buffer_ns": 78250.0137575, "command_buffers": 2 }, { "command_buffer_ns": 76916.7090766, "command_buffers": 2 }, { "command_buffer_ns": 111916.684546, "command_buffers": 2 }, { "command_buffer_ns": 114333.350211, "command_buffers": 2 }, { "command_buffer_ns": 110666.61682, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 80.46223956625, 82.91731774078124, 81.51562542484375, 79.6972653915625, 82.3430991659375, 82.7994781503125, 82.30403545890624, 119.53124976578125, 118.40039042 ], "command_buffer_throughput_us_p50": 82.3430991659375, "command_buffer_throughput_us_p90": 118.62656228915624, "command_buffer_latency_us": [ 113.458256237, 75.99999662489999, 76.3750285842, 106.916588265, 78.2500137575, 76.9167090766, 111.916684546, 114.333350211, 110.66661682 ], "command_buffer_latency_us_p50": 106.916588265, "command_buffer_latency_us_p90": 113.6332750318 }, "compute_throughput_us": [ 78.0234375, 80.00325, 79.962890625, 79.591140625, 82.46875, 77.29296875, 79.856765625, 81.87890625, 115.248703125 ], "compute_throughput_us_p50": 79.962890625, "compute_throughput_us_p90": 89.024740625, "command_buffer_throughput_us": [ 78.02669279046874, 80.00390698718749, 79.9641929915625, 79.59179583843749, 82.46940069515625, 77.293620051875, 79.85872343859374, 81.880208199375, 115.24934961921875 ], "command_buffer_throughput_us_p50": 79.9641929915625, "command_buffer_throughput_us_p90": 89.02539047996876, "compute_latency_us": [ 113.166, 116.166, 76.792, 343.25, 77.125, 111.5, 75.959, 77.416, 76.25 ], "compute_latency_us_p50": 77.416, "compute_latency_us_p90": 161.58280000000002, "command_buffer_latency_us": [ 113.20831254100001, 116.124982014, 76.8332974985, 343.333347701, 77.1666527726, 111.583329272, 76.0416733101, 77.37497799100001, 76.3332936913 ], "command_buffer_latency_us_p50": 77.37497799100001, "command_buffer_latency_us_p90": 161.56665515140003, "counter_control_throughput_ratio": 0.9702892816047468, "counter_control_latency_ratio": 0.9977970426808086 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 111.756726316, "throughput_us_p90": 113.7955550874, "latency_us_p50": 365.0, "latency_us_p90": 936.0496000000003 }, "torch": { "cold_call_ms": 7.145875, "warmup_ms": 201.251292, "repetitions": 102, "throughput_us": [ 296.2487745098039, 305.07884313725486, 280.6417549019608, 305.0453431372549, 305.09845098039216, 277.302294117647, 306.4313725490196, 283.2957549019608, 292.00407843137253 ], "latency_us": [ 592.5409999999999, 444.41700000000003, 434.541, 440.625, 449.875, 1207.333, 2242.0, 771.125, 534.417 ], "throughput_us_p50": 296.2487745098039, "throughput_us_p90": 305.36503529411766, "latency_us_p50": 534.417, "latency_us_p90": 1414.2664000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17762833.333108574, "command_buffers": 1 }, { "command_buffer_ns": 17562749.970238656, "command_buffers": 1 }, { "command_buffer_ns": 18091916.6887179, "command_buffers": 1 }, { "command_buffer_ns": 17521000.001579523, "command_buffers": 1 }, { "command_buffer_ns": 17540708.35886523, "command_buffers": 1 }, { "command_buffer_ns": 17501374.997664243, "command_buffers": 1 }, { "command_buffer_ns": 17812374.979257584, "command_buffers": 1 }, { "command_buffer_ns": 15116541.646420956, "command_buffers": 1 }, { "command_buffer_ns": 17681166.70427844, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 200666.6618399322, "command_buffers": 1 }, { "command_buffer_ns": 199124.9737329781, "command_buffers": 1 }, { "command_buffer_ns": 199041.62036255002, "command_buffers": 1 }, { "command_buffer_ns": 198416.70291498303, "command_buffers": 1 }, { "command_buffer_ns": 200958.3404287696, "command_buffers": 1 }, { "command_buffer_ns": 201083.37048441172, "command_buffers": 1 }, { "command_buffer_ns": 395875.0166930258, "command_buffers": 1 }, { "command_buffer_ns": 199750.00759586692, "command_buffers": 1 }, { "command_buffer_ns": 373625.0218935311, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 277.54427082982147, 274.417968284979, 282.6861982612172, 273.76562502468005, 274.07356810726924, 273.4589843385038, 278.31835905089974, 236.19596322532743, 276.2682297543506 ], "command_buffer_throughput_us_p50": 274.417968284979, "command_buffer_throughput_us_p90": 279.19192689296324, "command_buffer_latency_us": [ 200.6666618399322, 199.1249737329781, 199.04162036255002, 198.41670291498303, 200.9583404287696, 201.08337048441172, 395.8750166930258, 199.75000759586692, 373.6250218935311 ], "command_buffer_latency_us_p50": 200.6666618399322, "command_buffer_latency_us_p90": 378.07502085343003 }, "throughput": [ { "compute_ns": 17673792.0, "compute_span_ns": 17673792.0, "command_buffer_ns": 17673833.353910595, "calibration_cpu_ns": 18687666.0, "calibration_gpu_ticks": 18687666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17827083.0, "compute_span_ns": 17827083.0, "command_buffer_ns": 17827041.67999327, "calibration_cpu_ns": 19528583.0, "calibration_gpu_ticks": 19528583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17583875.0, "compute_span_ns": 17583875.0, "command_buffer_ns": 17583875.043783337, "calibration_cpu_ns": 18668875.0, "calibration_gpu_ticks": 18668875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17810584.0, "compute_span_ns": 17810584.0, "command_buffer_ns": 17810624.96593222, "calibration_cpu_ns": 20294542.0, "calibration_gpu_ticks": 20294542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17512000.0, "compute_span_ns": 17512000.0, "command_buffer_ns": 17511999.991256744, "calibration_cpu_ns": 19644791.0, "calibration_gpu_ticks": 19644791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17584167.0, "compute_span_ns": 17584167.0, "command_buffer_ns": 17584208.340849727, "calibration_cpu_ns": 19866375.0, "calibration_gpu_ticks": 19866375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17540208.0, "compute_span_ns": 17540208.0, "command_buffer_ns": 17540166.67837277, "calibration_cpu_ns": 18714500.0, "calibration_gpu_ticks": 18714500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17667584.0, "compute_span_ns": 17667584.0, "command_buffer_ns": 17667625.041212887, "calibration_cpu_ns": 19906333.0, "calibration_gpu_ticks": 19906333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17597875.0, "compute_span_ns": 17597875.0, "command_buffer_ns": 17597874.97576326, "calibration_cpu_ns": 18843917.0, "calibration_gpu_ticks": 18843917.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 201000.0, "compute_span_ns": 201000.0, "command_buffer_ns": 200999.95890632272, "calibration_cpu_ns": 721292.0, "calibration_gpu_ticks": 721292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199541.0, "compute_span_ns": 199541.0, "command_buffer_ns": 199500.00569224358, "calibration_cpu_ns": 817417.0, "calibration_gpu_ticks": 817417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200750.0, "compute_span_ns": 200750.0, "command_buffer_ns": 200750.0152103603, "calibration_cpu_ns": 684375.0, "calibration_gpu_ticks": 684375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 346375.0, "compute_span_ns": 346375.0, "command_buffer_ns": 346374.9890215695, "calibration_cpu_ns": 1712541.0, "calibration_gpu_ticks": 1712541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 314209.0, "compute_span_ns": 314209.0, "command_buffer_ns": 314250.0063404441, "calibration_cpu_ns": 1870000.0, "calibration_gpu_ticks": 1870000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 197500.0, "compute_span_ns": 197500.0, "command_buffer_ns": 197499.99046325684, "calibration_cpu_ns": 625292.0, "calibration_gpu_ticks": 625292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 197542.0, "compute_span_ns": 197542.0, "command_buffer_ns": 197541.66714847088, "calibration_cpu_ns": 886417.0, "calibration_gpu_ticks": 886417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201291.0, "compute_span_ns": 201291.0, "command_buffer_ns": 201250.01901760697, "calibration_cpu_ns": 723792.0, "calibration_gpu_ticks": 723792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 197334.0, "compute_span_ns": 197334.0, "command_buffer_ns": 197375.01861527562, "calibration_cpu_ns": 881000.0, "calibration_gpu_ticks": 881000.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 276.153, 278.548171875, 274.748046875, 278.290375, 273.625, 274.752609375, 274.06575, 276.056, 274.966796875 ], "compute_throughput_us_p50": 274.966796875, "compute_throughput_us_p90": 278.341934375, "command_buffer_throughput_us": [ 276.15364615485305, 278.5475262498949, 274.74804755911464, 278.29101509269094, 273.6249998633866, 274.753255325777, 274.06510434957454, 276.05664126895135, 274.96679649630096 ], "command_buffer_throughput_us_p50": 274.96679649630096, "command_buffer_throughput_us_p90": 278.3423173241317, "compute_latency_us": [ 201.0, 199.541, 200.75, 346.375, 314.209, 197.5, 197.542, 201.291, 197.334 ], "compute_latency_us_p50": 200.75, "compute_latency_us_p90": 320.6422, "command_buffer_latency_us": [ 200.99995890632272, 199.50000569224358, 200.7500152103603, 346.3749890215695, 314.2500063404441, 197.49999046325684, 197.54166714847088, 201.25001901760697, 197.37501861527562 ], "command_buffer_latency_us_p50": 200.7500152103603, "command_buffer_latency_us_p90": 320.67500287666917, "counter_control_throughput_ratio": 0.9983633290616808, "counter_control_latency_ratio": 1.0018833999183265 }, "download_ms": 0.717417, "allocation_upload_ms": 17.361042, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3772394552548658, "device_compute_slowdown": 0.2908092596407237, "valid": true, "round": 0, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.540208, "warmup_ms": 201.694375, "repetitions": 115, "throughput_us": [ 293.6840608695652, 291.1181217391304, 292.370652173913, 289.139852173913, 292.1231913043478, 290.91413043478263, 301.2369565217391, 293.94347826086954, 279.6902173913044 ], "latency_us": [ 3266.791, 1013.833, 499.667, 468.125, 447.625, 442.083, 435.459, 445.0, 452.667 ], "throughput_us_p50": 292.1231913043478, "throughput_us_p90": 295.40217391304344, "latency_us_p50": 452.667, "latency_us_p90": 1464.4246000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17582541.680894792, "command_buffers": 1 }, { "command_buffer_ns": 17721875.046845526, "command_buffers": 1 }, { "command_buffer_ns": 17533000.034745783, "command_buffers": 1 }, { "command_buffer_ns": 17618541.66397825, "command_buffers": 1 }, { "command_buffer_ns": 17534375.01611188, "command_buffers": 1 }, { "command_buffer_ns": 15188291.668891907, "command_buffers": 1 }, { "command_buffer_ns": 17569124.98967722, "command_buffers": 1 }, { "command_buffer_ns": 17677875.00284612, "command_buffers": 1 }, { "command_buffer_ns": 17616583.325434476, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 198916.6485145688, "command_buffers": 1 }, { "command_buffer_ns": 200624.98515471816, "command_buffers": 1 }, { "command_buffer_ns": 199208.3271034062, "command_buffers": 1 }, { "command_buffer_ns": 199541.68237745762, "command_buffers": 1 }, { "command_buffer_ns": 413500.0053793192, "command_buffers": 1 }, { "command_buffer_ns": 200500.01330673695, "command_buffers": 1 }, { "command_buffer_ns": 457500.0493787229, "command_buffers": 1 }, { "command_buffer_ns": 200250.0114031136, "command_buffers": 1 }, { "command_buffer_ns": 343375.0243857503, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 274.7272137639811, 276.90429760696134, 273.95312554290285, 275.28971349966014, 273.97460962674813, 237.31705732643604, 274.5175779637066, 276.21679691947065, 275.2591144599137 ], "command_buffer_throughput_us_p50": 274.7272137639811, "command_buffer_throughput_us_p90": 276.3542970569688, "command_buffer_latency_us": [ 198.9166485145688, 200.62498515471816, 199.2083271034062, 199.54168237745762, 413.5000053793192, 200.50001330673695, 457.5000493787229, 200.2500114031136, 343.3750243857503 ], "command_buffer_latency_us_p50": 200.50001330673695, "command_buffer_latency_us_p90": 422.30001417919993 }, "throughput": [ { "compute_ns": 17524417.0, "compute_span_ns": 17524417.0, "command_buffer_ns": 17524458.293337375, "calibration_cpu_ns": 18591792.0, "calibration_gpu_ticks": 18591792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18474875.0, "compute_span_ns": 18474875.0, "command_buffer_ns": 18474875.018000603, "calibration_cpu_ns": 20139250.0, "calibration_gpu_ticks": 20139250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17773875.0, "compute_span_ns": 17773875.0, "command_buffer_ns": 17773875.035345554, "calibration_cpu_ns": 19491708.0, "calibration_gpu_ticks": 19491708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17619875.0, "compute_span_ns": 17619875.0, "command_buffer_ns": 17619874.968659133, "calibration_cpu_ns": 18622500.0, "calibration_gpu_ticks": 18622500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17686500.0, "compute_span_ns": 17686500.0, "command_buffer_ns": 17686499.981209636, "calibration_cpu_ns": 20070666.0, "calibration_gpu_ticks": 20070666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17855208.0, "compute_span_ns": 17855208.0, "command_buffer_ns": 17855166.632216424, "calibration_cpu_ns": 19343458.0, "calibration_gpu_ticks": 19343458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 15110875.0, "compute_span_ns": 15110875.0, "command_buffer_ns": 15110874.956008047, "calibration_cpu_ns": 16267209.0, "calibration_gpu_ticks": 16267209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17610958.0, "compute_span_ns": 17610958.0, "command_buffer_ns": 17610916.635021567, "calibration_cpu_ns": 19057958.0, "calibration_gpu_ticks": 19057958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17523750.0, "compute_span_ns": 17523750.0, "command_buffer_ns": 17523750.02251938, "calibration_cpu_ns": 18520542.0, "calibration_gpu_ticks": 18520542.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 199750.0, "compute_span_ns": 199750.0, "command_buffer_ns": 199750.00759586692, "calibration_cpu_ns": 874750.0, "calibration_gpu_ticks": 874750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199292.0, "compute_span_ns": 199292.0, "command_buffer_ns": 199291.68047383428, "calibration_cpu_ns": 1551917.0, "calibration_gpu_ticks": 1551917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 556833.0, "compute_span_ns": 556833.0, "command_buffer_ns": 556833.3435803652, "calibration_cpu_ns": 2006125.0, "calibration_gpu_ticks": 2006125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 196958.0, "compute_span_ns": 196958.0, "command_buffer_ns": 196958.3099707961, "calibration_cpu_ns": 1071792.0, "calibration_gpu_ticks": 1071792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 197875.0, "compute_span_ns": 197875.0, "command_buffer_ns": 197874.9642148614, "calibration_cpu_ns": 794583.0, "calibration_gpu_ticks": 794583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201916.0, "compute_span_ns": 201916.0, "command_buffer_ns": 201874.99467283487, "calibration_cpu_ns": 635417.0, "calibration_gpu_ticks": 635417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201375.0, "compute_span_ns": 201375.0, "command_buffer_ns": 201374.9908655882, "calibration_cpu_ns": 1067833.0, "calibration_gpu_ticks": 1067833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201291.0, "compute_span_ns": 201291.0, "command_buffer_ns": 201250.01901760697, "calibration_cpu_ns": 619542.0, "calibration_gpu_ticks": 619542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 196750.0, "compute_span_ns": 196750.0, "command_buffer_ns": 196749.9847523868, "calibration_cpu_ns": 836541.0, "calibration_gpu_ticks": 836541.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 273.819015625, 288.669921875, 277.716796875, 275.310546875, 276.3515625, 278.987625, 236.107421875, 275.17121875, 273.80859375 ], "compute_throughput_us_p50": 275.310546875, "compute_throughput_us_p90": 280.924084375, "command_buffer_throughput_us": [ 273.8196608333965, 288.6699221562594, 277.7167974272743, 275.31054638529895, 276.35156220640056, 278.9869786283816, 236.10742118762573, 275.170572422212, 273.8085941018653 ], "command_buffer_throughput_us_p50": 275.31054638529895, "command_buffer_throughput_us_p90": 280.9235673339572, "compute_latency_us": [ 199.75, 199.292, 556.833, 196.958, 197.875, 201.916, 201.375, 201.291, 196.75 ], "compute_latency_us_p50": 199.75, "compute_latency_us_p90": 272.89940000000007, "command_buffer_latency_us": [ 199.75000759586692, 199.29168047383428, 556.8333435803652, 196.9583099707961, 197.8749642148614, 201.87499467283487, 201.3749908655882, 201.25001901760697, 196.7499847523868 ], "command_buffer_latency_us_p50": 199.75000759586692, "command_buffer_latency_us_p90": 272.866664454341, "counter_control_throughput_ratio": 1.000075676222602, "counter_control_latency_ratio": 0.9933542440895102 }, "download_ms": 0.625125, "allocation_upload_ms": 1.305, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-w7d01w4p/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "6caff4b0a979c3104fea9f23f0d5430392f6f4e7d743dab03e9fdb6de1428bf3", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.829, "capture_ms": 0.070042, "compile_ms": 64.294209, "allocation_upload_ms": 7.037167, "cold_call_ms": 5.921334, "warmup_ms": 200.98675, "download_ms": 3.187459, "repetitions": 432, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 92.3878287037, 89.0200625, 89.0532407407, 92.7531828704, 94.0170717593, 87.2567523148, 92.3525277778, 93.4195601852, 93.481 ], "latency_us": [ 857.042, 399.709, 324.375, 291.125, 295.417, 296.709, 2829.667, 716.542, 397.708 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4184625, "compute_span_ns": 4184625, "calibration_cpu_ns": 5537458, "calibration_gpu_ticks": 5537458, "compute_passes": 1, "command_buffer_ns": 4184708.29027, "command_buffers": 2 }, { "compute_ns": 4173875, "compute_span_ns": 4173875, "calibration_cpu_ns": 5350375, "calibration_gpu_ticks": 5350375, "compute_passes": 1, "command_buffer_ns": 4173874.97146, "command_buffers": 2 }, { "compute_ns": 4415959, "compute_span_ns": 4415959, "calibration_cpu_ns": 5641542, "calibration_gpu_ticks": 5641542, "compute_passes": 1, "command_buffer_ns": 4416124.95342, "command_buffers": 2 }, { "compute_ns": 4308209, "compute_span_ns": 4308209, "calibration_cpu_ns": 5645458, "calibration_gpu_ticks": 5645458, "compute_passes": 1, "command_buffer_ns": 4308291.65271, "command_buffers": 2 }, { "compute_ns": 4488250, "compute_span_ns": 4488250, "calibration_cpu_ns": 5641500, "calibration_gpu_ticks": 5641500, "compute_passes": 1, "command_buffer_ns": 4488291.68454, "command_buffers": 2 }, { "compute_ns": 4204959, "compute_span_ns": 4204959, "calibration_cpu_ns": 5820291, "calibration_gpu_ticks": 5820291, "compute_passes": 1, "command_buffer_ns": 4205124.97658, "command_buffers": 2 }, { "compute_ns": 4349042, "compute_span_ns": 4349042, "calibration_cpu_ns": 5471250, "calibration_gpu_ticks": 5471250, "compute_passes": 1, "command_buffer_ns": 4349083.34864, "command_buffers": 2 }, { "compute_ns": 6544334, "compute_span_ns": 6544334, "calibration_cpu_ns": 7371875, "calibration_gpu_ticks": 7371875, "compute_passes": 1, "command_buffer_ns": 6544416.65392, "command_buffers": 2 }, { "compute_ns": 6616750, "compute_span_ns": 6616750, "calibration_cpu_ns": 7778458, "calibration_gpu_ticks": 7778458, "compute_passes": 1, "command_buffer_ns": 6616791.71026, "command_buffers": 2 } ], "latency": [ { "compute_ns": 63167, "compute_span_ns": 63167, "calibration_cpu_ns": 481583, "calibration_gpu_ticks": 481583, "compute_passes": 1, "command_buffer_ns": 63250.0159554, "command_buffers": 2 }, { "compute_ns": 331166, "compute_span_ns": 331166, "calibration_cpu_ns": 1542709, "calibration_gpu_ticks": 1542709, "compute_passes": 1, "command_buffer_ns": 331208.400894, "command_buffers": 2 }, { "compute_ns": 63584, "compute_span_ns": 63584, "calibration_cpu_ns": 563625, "calibration_gpu_ticks": 563625, "compute_passes": 1, "command_buffer_ns": 63708.2848698, "command_buffers": 2 }, { "compute_ns": 104208, "compute_span_ns": 104208, "calibration_cpu_ns": 1717542, "calibration_gpu_ticks": 1717542, "compute_passes": 1, "command_buffer_ns": 104208.302218, "command_buffers": 2 }, { "compute_ns": 107417, "compute_span_ns": 107417, "calibration_cpu_ns": 1140292, "calibration_gpu_ticks": 1140292, "compute_passes": 1, "command_buffer_ns": 107458.326966, "command_buffers": 2 }, { "compute_ns": 63583, "compute_span_ns": 63583, "calibration_cpu_ns": 634459, "calibration_gpu_ticks": 634459, "compute_passes": 1, "command_buffer_ns": 63583.3130218, "command_buffers": 2 }, { "compute_ns": 63459, "compute_span_ns": 63459, "calibration_cpu_ns": 580209, "calibration_gpu_ticks": 580209, "compute_passes": 1, "command_buffer_ns": 63541.6363366, "command_buffers": 2 }, { "compute_ns": 103959, "compute_span_ns": 103959, "calibration_cpu_ns": 725167, "calibration_gpu_ticks": 725167, "compute_passes": 1, "command_buffer_ns": 104000.035208, "command_buffers": 2 }, { "compute_ns": 63250, "compute_span_ns": 63250, "calibration_cpu_ns": 483000, "calibration_gpu_ticks": 483000, "compute_passes": 1, "command_buffer_ns": 63291.634433, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4232624.95315, "command_buffers": 2 }, { "command_buffer_ns": 4391500.02785, "command_buffers": 2 }, { "command_buffer_ns": 4298874.93374, "command_buffers": 2 }, { "command_buffer_ns": 4326958.26748, "command_buffers": 2 }, { "command_buffer_ns": 4355541.60504, "command_buffers": 2 }, { "command_buffer_ns": 4334249.99937, "command_buffers": 2 }, { "command_buffer_ns": 4481874.98841, "command_buffers": 2 }, { "command_buffer_ns": 3971083.38075, "command_buffers": 2 }, { "command_buffer_ns": 4183999.96124, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 66791.6610837, "command_buffers": 2 }, { "command_buffer_ns": 97916.694358, "command_buffers": 2 }, { "command_buffer_ns": 63624.9314994, "command_buffers": 2 }, { "command_buffer_ns": 63333.3111182, "command_buffers": 2 }, { "command_buffer_ns": 104791.659396, "command_buffers": 2 }, { "command_buffer_ns": 243666.698225, "command_buffers": 2 }, { "command_buffer_ns": 102208.345197, "command_buffers": 2 }, { "command_buffer_ns": 63541.6363366, "command_buffers": 2 }, { "command_buffer_ns": 108666.718006, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 66.13476489296876, 68.61718793515625, 67.1699208396875, 67.608722929375, 68.05533757875, 67.72265624015625, 70.02929669390625, 62.04817782421875, 65.374999394375 ], "command_buffer_throughput_us_p50": 67.608722929375, "command_buffer_throughput_us_p90": 68.89960968690625, "command_buffer_latency_us": [ 66.7916610837, 97.91669435799999, 63.6249314994, 63.3333111182, 104.791659396, 243.666698225, 102.208345197, 63.5416363366, 108.666718006 ], "command_buffer_latency_us_p50": 97.91669435799999, "command_buffer_latency_us_p90": 135.66671404980002 }, "compute_throughput_us": [ 65.384765625, 65.216796875, 68.999359375, 67.315765625, 70.12890625, 65.702484375, 67.95378125, 102.25521875, 103.38671875 ], "compute_throughput_us_p50": 67.95378125, "compute_throughput_us_p90": 102.48151874999999, "command_buffer_throughput_us": [ 65.38606703546876, 65.2167964290625, 69.00195239718751, 67.31705707359374, 70.1295575709375, 65.70507775906249, 67.95442732250001, 102.25651021750001, 103.3873704728125 ], "command_buffer_throughput_us_p50": 67.95442732250001, "command_buffer_throughput_us_p90": 102.48268226856251, "compute_latency_us": [ 63.167, 331.166, 63.584, 104.208, 107.417, 63.583, 63.459, 103.959, 63.25 ], "compute_latency_us_p50": 63.584, "compute_latency_us_p90": 152.16680000000002, "command_buffer_latency_us": [ 63.250015955399995, 331.208400894, 63.7082848698, 104.208302218, 107.45832696599999, 63.5833130218, 63.5416363366, 104.000035208, 63.291634433 ], "command_buffer_latency_us_p50": 63.7082848698, "command_buffer_latency_us_p90": 152.20834175160002, "counter_control_throughput_ratio": 0.9956859730054962, "counter_control_latency_ratio": 1.0013100740297187 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 92.3878287037, "throughput_us_p90": 93.58821435185999, "latency_us_p50": 397.708, "latency_us_p90": 1251.5670000000005 }, "slowdown": 0.31626324596545285, "device_compute_slowdown": 0.24682592810675447, "valid": true, "round": 0, "variant": "reference" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-y15zbm72/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "0d8a82e58cbe97c43d15ef559cf043ad575745e0d3fb0f4f40ed60b2caca21c0", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.936708, "capture_ms": 0.07075, "compile_ms": 54.144209, "allocation_upload_ms": 1.759708, "cold_call_ms": 7.340375, "warmup_ms": 200.163125, "download_ms": 0.91275, "repetitions": 1238, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 19.8515751212, 19.8722399031, 19.8460549273, 20.1513861066, 20.7755783522, 19.8934442649, 19.8677641357, 19.7341147011, 19.3713311793 ], "latency_us": [ 292.917, 421.041, 374.25, 210.375, 230, 315.042, 219.209, 207.958, 240.583 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 879958, "compute_span_ns": 879958, "calibration_cpu_ns": 1345000, "calibration_gpu_ticks": 1345000, "compute_passes": 1, "command_buffer_ns": 879999.948665, "command_buffers": 2 }, { "compute_ns": 1310000, "compute_span_ns": 1310000, "calibration_cpu_ns": 2382208, "calibration_gpu_ticks": 2382208, "compute_passes": 1, "command_buffer_ns": 1310083.37485, "command_buffers": 2 }, { "compute_ns": 859333, "compute_span_ns": 859333, "calibration_cpu_ns": 1425042, "calibration_gpu_ticks": 1425042, "compute_passes": 1, "command_buffer_ns": 859333.318658, "command_buffers": 2 }, { "compute_ns": 1314708, "compute_span_ns": 1314708, "calibration_cpu_ns": 2718916, "calibration_gpu_ticks": 2718916, "compute_passes": 1, "command_buffer_ns": 1314791.67612, "command_buffers": 2 }, { "compute_ns": 854792, "compute_span_ns": 854792, "calibration_cpu_ns": 1343208, "calibration_gpu_ticks": 1343208, "compute_passes": 1, "command_buffer_ns": 854875.019286, "command_buffers": 2 }, { "compute_ns": 1277167, "compute_span_ns": 1277167, "calibration_cpu_ns": 2244458, "calibration_gpu_ticks": 2244458, "compute_passes": 1, "command_buffer_ns": 1277250.00493, "command_buffers": 2 }, { "compute_ns": 863042, "compute_span_ns": 863042, "calibration_cpu_ns": 1379916, "calibration_gpu_ticks": 1379916, "compute_passes": 1, "command_buffer_ns": 863166.584168, "command_buffers": 2 }, { "compute_ns": 1378292, "compute_span_ns": 1378292, "calibration_cpu_ns": 2804334, "calibration_gpu_ticks": 2804334, "compute_passes": 1, "command_buffer_ns": 1378374.98914, "command_buffers": 2 }, { "compute_ns": 855041, "compute_span_ns": 855041, "calibration_cpu_ns": 1429000, "calibration_gpu_ticks": 1429000, "compute_passes": 1, "command_buffer_ns": 855083.344504, "command_buffers": 2 } ], "latency": [ { "compute_ns": 17916, "compute_span_ns": 17916, "calibration_cpu_ns": 835916, "calibration_gpu_ticks": 835916, "compute_passes": 1, "command_buffer_ns": 17999.9624379, "command_buffers": 2 }, { "compute_ns": 15917, "compute_span_ns": 15917, "calibration_cpu_ns": 411792, "calibration_gpu_ticks": 411792, "compute_passes": 1, "command_buffer_ns": 16041.6238941, "command_buffers": 2 }, { "compute_ns": 94000, "compute_span_ns": 94000, "calibration_cpu_ns": 1088083, "calibration_gpu_ticks": 1088083, "compute_passes": 1, "command_buffer_ns": 94041.6939557, "command_buffers": 2 }, { "compute_ns": 19500, "compute_span_ns": 19500, "calibration_cpu_ns": 890542, "calibration_gpu_ticks": 890542, "compute_passes": 1, "command_buffer_ns": 19625.0039153, "command_buffers": 2 }, { "compute_ns": 54500, "compute_span_ns": 54500, "calibration_cpu_ns": 1113041, "calibration_gpu_ticks": 1113041, "compute_passes": 1, "command_buffer_ns": 54541.6842215, "command_buffers": 2 }, { "compute_ns": 20000, "compute_span_ns": 20000, "calibration_cpu_ns": 834750, "calibration_gpu_ticks": 834750, "compute_passes": 1, "command_buffer_ns": 20000.0358745, "command_buffers": 2 }, { "compute_ns": 18042, "compute_span_ns": 18042, "calibration_cpu_ns": 419917, "calibration_gpu_ticks": 419917, "compute_passes": 1, "command_buffer_ns": 18166.6691788, "command_buffers": 2 }, { "compute_ns": 73791, "compute_span_ns": 73791, "calibration_cpu_ns": 992750, "calibration_gpu_ticks": 992750, "compute_passes": 1, "command_buffer_ns": 73833.274655, "command_buffers": 2 }, { "compute_ns": 16000, "compute_span_ns": 16000, "calibration_cpu_ns": 453208, "calibration_gpu_ticks": 453208, "compute_passes": 1, "command_buffer_ns": 16083.358787, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1101583.30062, "command_buffers": 2 }, { "command_buffer_ns": 873749.959283, "command_buffers": 2 }, { "command_buffer_ns": 1229333.34205, "command_buffers": 2 }, { "command_buffer_ns": 858249.957673, "command_buffers": 2 }, { "command_buffer_ns": 1257500.02917, "command_buffers": 2 }, { "command_buffer_ns": 870999.99655, "command_buffers": 2 }, { "command_buffer_ns": 1375125.02261, "command_buffers": 2 }, { "command_buffer_ns": 863333.349116, "command_buffers": 2 }, { "command_buffer_ns": 1325083.37265, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 110208.289698, "command_buffers": 2 }, { "command_buffer_ns": 17833.3139047, "command_buffers": 2 }, { "command_buffer_ns": 18083.374016, "command_buffers": 2 }, { "command_buffer_ns": 15791.6801982, "command_buffers": 2 }, { "command_buffer_ns": 18166.6691788, "command_buffers": 2 }, { "command_buffer_ns": 15791.6219905, "command_buffers": 2 }, { "command_buffer_ns": 18374.9943972, "command_buffers": 2 }, { "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "command_buffer_ns": 16041.6821018, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 17.2122390721875, 13.652343113796874, 19.20833346953125, 13.410155588640626, 19.64843795578125, 13.60937494609375, 21.48632847828125, 13.4895835799375, 20.70442769765625 ], "command_buffer_throughput_us_p50": 17.2122390721875, "command_buffer_throughput_us_p90": 20.86080785378125, "command_buffer_latency_us": [ 110.20828969799999, 17.8333139047, 18.083374016, 15.7916801982, 18.1666691788, 15.7916219905, 18.374994397200002, 20.125007722499998, 16.0416821018 ], "command_buffer_latency_us_p50": 18.083374016, "command_buffer_latency_us_p90": 38.141664117600016 }, "compute_throughput_us": [ 13.74934375, 20.46875, 13.427078125, 20.5423125, 13.356125, 19.955734375, 13.48503125, 21.5358125, 13.360015625 ], "compute_throughput_us_p50": 13.74934375, "compute_throughput_us_p90": 20.7410125, "command_buffer_throughput_us": [ 13.749999197890626, 20.47005273203125, 13.427083104031249, 20.543619939375, 13.35742217634375, 19.957031327031252, 13.486977877625, 21.5371092053125, 13.360677257875 ], "command_buffer_throughput_us_p50": 13.749999197890626, "command_buffer_throughput_us_p90": 20.7423177925625, "compute_latency_us": [ 17.916, 15.917, 94.0, 19.5, 54.5, 20.0, 18.042, 73.791, 16.0 ], "compute_latency_us_p50": 19.5, "compute_latency_us_p90": 77.8328, "command_buffer_latency_us": [ 17.999962437900002, 16.0416238941, 94.0416939557, 19.6250039153, 54.5416842215, 20.0000358745, 18.1666691788, 73.833274655, 16.083358786999998 ], "command_buffer_latency_us_p50": 19.6250039153, "command_buffer_latency_us_p90": 77.87495851514, "counter_control_throughput_ratio": 0.7988501170721387, "counter_control_latency_ratio": 1.2427432463796309 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 19.8677641357, "throughput_us_p90": 20.27622455572, "latency_us_p50": 240.583, "latency_us_p90": 383.6082 }, "torch": { "cold_call_ms": 10.773458, "warmup_ms": 200.732458, "repetitions": 711, "throughput_us": [ 46.06000984528832, 46.20739662447258, 45.99185372714487, 42.79395218002813, 44.67035864978903, 45.772971870604785, 46.16789732770746, 46.016994374120955, 46.00914205344585 ], "latency_us": [ 520.792, 311.209, 361.917, 318.583, 354.5, 344.167, 323.542, 337.583, 347.45799999999997 ], "throughput_us_p50": 46.00914205344585, "throughput_us_p90": 46.17579718706048, "latency_us_p50": 344.167, "latency_us_p90": 393.692, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1914208.2892358303, "command_buffers": 11 }, { "command_buffer_ns": 1929333.2006782293, "command_buffers": 11 }, { "command_buffer_ns": 2029958.413913846, "command_buffers": 11 }, { "command_buffer_ns": 1943208.2772254944, "command_buffers": 11 }, { "command_buffer_ns": 1974333.48512277, "command_buffers": 11 }, { "command_buffer_ns": 1944625.0516921282, "command_buffers": 11 }, { "command_buffer_ns": 1880791.5039360523, "command_buffers": 11 }, { "command_buffer_ns": 1934541.6221767664, "command_buffers": 11 }, { "command_buffer_ns": 1933958.2649990916, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 65750.03499165177, "command_buffers": 1 }, { "command_buffer_ns": 47624.98429045081, "command_buffers": 1 }, { "command_buffer_ns": 154750.01418963075, "command_buffers": 1 }, { "command_buffer_ns": 48250.018153339624, "command_buffers": 1 }, { "command_buffer_ns": 84458.32645520568, "command_buffers": 1 }, { "command_buffer_ns": 57791.70896857977, "command_buffers": 1 }, { "command_buffer_ns": 47208.33385363221, "command_buffers": 1 }, { "command_buffer_ns": 46583.35819840431, "command_buffers": 1 }, { "command_buffer_ns": 47750.01434609294, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.90950451930985, 30.145831260597333, 31.718100217403844, 30.36262933164835, 30.848960705043282, 30.384766432689503, 29.387367249000818, 30.227212846511975, 30.218097890610807 ], "command_buffer_throughput_us_p50": 30.227212846511975, "command_buffer_throughput_us_p90": 31.022788607515395, "command_buffer_latency_us": [ 65.75003499165177, 47.62498429045081, 154.75001418963075, 48.250018153339624, 84.45832645520568, 57.79170896857977, 47.20833385363221, 46.58335819840431, 47.75001434609294 ], "command_buffer_latency_us_p50": 48.250018153339624, "command_buffer_latency_us_p90": 98.5166640020907 }, "throughput": [ { "compute_ns": 3068088.0, "compute_span_ns": 4148250.0, "command_buffer_ns": 3774749.9882243574, "calibration_cpu_ns": 6008875.0, "calibration_gpu_ticks": 6008875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3028083.0, "compute_span_ns": 4148583.0, "command_buffer_ns": 3996000.04311651, "calibration_cpu_ns": 5471167.0, "calibration_gpu_ticks": 5471167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3001501.0, "compute_span_ns": 4355916.0, "command_buffer_ns": 4198333.3067037165, "calibration_cpu_ns": 6089875.0, "calibration_gpu_ticks": 6089875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3039175.0, "compute_span_ns": 3931417.0, "command_buffer_ns": 3793208.335991949, "calibration_cpu_ns": 5295750.0, "calibration_gpu_ticks": 5295750.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3030503.0, "compute_span_ns": 4149458.0, "command_buffer_ns": 3994333.4413692355, "calibration_cpu_ns": 5907541.0, "calibration_gpu_ticks": 5907541.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2993081.0, "compute_span_ns": 4182125.0, "command_buffer_ns": 4013583.2969099283, "calibration_cpu_ns": 5883000.0, "calibration_gpu_ticks": 5883000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3030000.0, "compute_span_ns": 4120667.0, "command_buffer_ns": 3955166.6122861207, "calibration_cpu_ns": 5730292.0, "calibration_gpu_ticks": 5730292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3043084.0, "compute_span_ns": 4081417.0, "command_buffer_ns": 3824166.429694742, "calibration_cpu_ns": 5548042.0, "calibration_gpu_ticks": 5548042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3075999.0, "compute_span_ns": 3943250.0, "command_buffer_ns": 3801333.368755877, "calibration_cpu_ns": 5822750.0, "calibration_gpu_ticks": 5822750.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 48416.0, "compute_span_ns": 48416.0, "command_buffer_ns": 48374.99000132084, "calibration_cpu_ns": 960125.0, "calibration_gpu_ticks": 960125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47334.0, "compute_span_ns": 47334.0, "command_buffer_ns": 47374.98238682747, "calibration_cpu_ns": 591208.0, "calibration_gpu_ticks": 591208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49583.0, "compute_span_ns": 49583.0, "command_buffer_ns": 49583.32283422351, "calibration_cpu_ns": 795084.0, "calibration_gpu_ticks": 795084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46958.0, "compute_span_ns": 46958.0, "command_buffer_ns": 46916.71347245574, "calibration_cpu_ns": 584209.0, "calibration_gpu_ticks": 584209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47500.0, "compute_span_ns": 47500.0, "command_buffer_ns": 47499.95423480868, "calibration_cpu_ns": 758125.0, "calibration_gpu_ticks": 758125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47708.0, "compute_span_ns": 47708.0, "command_buffer_ns": 47708.3376608789, "calibration_cpu_ns": 583667.0, "calibration_gpu_ticks": 583667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47875.0, "compute_span_ns": 47875.0, "command_buffer_ns": 47875.04440173507, "calibration_cpu_ns": 674791.0, "calibration_gpu_ticks": 674791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 51208.0, "compute_span_ns": 51208.0, "command_buffer_ns": 51166.68762639165, "calibration_cpu_ns": 2360875.0, "calibration_gpu_ticks": 2360875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 52375.0, "compute_span_ns": 52375.0, "command_buffer_ns": 52375.02045929432, "calibration_cpu_ns": 2462750.0, "calibration_gpu_ticks": 2462750.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.938875, 47.313796875, 46.898453125, 47.487109375, 47.351609375, 46.766890625, 47.34375, 47.5481875, 48.062484375 ], "compute_throughput_us_p50": 47.351609375, "compute_throughput_us_p90": 47.963596875, "command_buffer_throughput_us": [ 58.980468566005584, 62.43750067369547, 65.59895791724557, 59.2688802498742, 62.411460021394305, 62.71223901421763, 61.799478316970635, 59.75260046398034, 59.39583388681058 ], "command_buffer_throughput_us_p50": 61.799478316970635, "command_buffer_throughput_us_p90": 63.28958279482322, "compute_latency_us": [ 48.416, 47.334, 49.583, 46.958, 47.5, 47.708, 47.875, 51.208, 52.375 ], "compute_latency_us_p50": 47.875, "compute_latency_us_p90": 51.4414, "command_buffer_latency_us": [ 48.37499000132084, 47.37498238682747, 49.58332283422351, 46.91671347245574, 47.49995423480868, 47.7083376608789, 47.87504440173507, 51.16668762639165, 52.37502045929432 ], "command_buffer_latency_us_p50": 47.87504440173507, "command_buffer_latency_us_p90": 51.40835419297218, "counter_control_throughput_ratio": 2.0231300697008923, "counter_control_latency_ratio": 0.9723667527616962 }, "download_ms": 0.482583, "allocation_upload_ms": 1.167417, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.43182209554398776, "device_compute_slowdown": 0.2903669786830767, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.521167, "warmup_ms": 201.495167, "repetitions": 571, "throughput_us": [ 44.8206357267951, 45.45913485113836, 45.096322241681264, 43.32100175131348, 43.92155516637479, 45.07428371278459, 45.00211558669002, 45.88178633975482, 44.64674605954466 ], "latency_us": [ 510.583, 351.66700000000003, 334.541, 323.167, 311.917, 328.70899999999995, 382.91700000000003, 349.625, 314.0 ], "throughput_us_p50": 45.00211558669002, "throughput_us_p90": 45.54366514886165, "latency_us_p50": 334.541, "latency_us_p90": 408.45020000000005, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1900000.0320374966, "command_buffers": 11 }, { "command_buffer_ns": 1933791.6164658964, "command_buffers": 11 }, { "command_buffer_ns": 1967916.6143760085, "command_buffers": 11 }, { "command_buffer_ns": 1930041.5879115462, "command_buffers": 11 }, { "command_buffer_ns": 1937500.0847503543, "command_buffers": 11 }, { "command_buffer_ns": 1908625.1850239933, "command_buffers": 11 }, { "command_buffer_ns": 1928000.128827989, "command_buffers": 11 }, { "command_buffer_ns": 1966999.9601319432, "command_buffers": 11 }, { "command_buffer_ns": 1907833.3862125874, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 52249.99040365219, "command_buffers": 1 }, { "command_buffer_ns": 65375.0030323863, "command_buffers": 1 }, { "command_buffer_ns": 50249.97517466545, "command_buffers": 1 }, { "command_buffer_ns": 124500.01668184996, "command_buffers": 1 }, { "command_buffer_ns": 48291.63663089275, "command_buffers": 1 }, { "command_buffer_ns": 62624.982092529535, "command_buffers": 1 }, { "command_buffer_ns": 51791.66328161955, "command_buffers": 1 }, { "command_buffer_ns": 145166.70489683747, "command_buffers": 1 }, { "command_buffer_ns": 49500.02767145634, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.687500500585884, 30.21549400727963, 30.748697099625133, 30.15689981111791, 30.273438824224286, 29.822268515999895, 30.12500201293733, 30.734374377061613, 29.809896659571677 ], "command_buffer_throughput_us_p50": 30.15689981111791, "command_buffer_throughput_us_p90": 30.737238921574317, "command_buffer_latency_us": [ 52.24999040365219, 65.3750030323863, 50.24997517466545, 124.50001668184996, 48.291636630892754, 62.624982092529535, 51.79166328161955, 145.16670489683747, 49.50002767145634 ], "command_buffer_latency_us_p50": 52.24999040365219, "command_buffer_latency_us_p90": 128.63335432484746 }, "throughput": [ { "compute_ns": 2998250.0, "compute_span_ns": 4190417.0, "command_buffer_ns": 3906333.411578089, "calibration_cpu_ns": 5758042.0, "calibration_gpu_ticks": 5758042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3038873.0, "compute_span_ns": 4012709.0, "command_buffer_ns": 3877833.3691880107, "calibration_cpu_ns": 5085083.0, "calibration_gpu_ticks": 5085083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3017998.0, "compute_span_ns": 4334333.0, "command_buffer_ns": 3994458.355009556, "calibration_cpu_ns": 5457166.0, "calibration_gpu_ticks": 5457166.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2985168.0, "compute_span_ns": 4153042.0, "command_buffer_ns": 4006166.651379317, "calibration_cpu_ns": 5469792.0, "calibration_gpu_ticks": 5469792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3011582.0, "compute_span_ns": 3829458.0, "command_buffer_ns": 3703374.939505011, "calibration_cpu_ns": 6347916.0, "calibration_gpu_ticks": 6347916.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3095584.0, "compute_span_ns": 4213250.0, "command_buffer_ns": 3954000.014346093, "calibration_cpu_ns": 5329834.0, "calibration_gpu_ticks": 5329834.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3037713.0, "compute_span_ns": 4059292.0, "command_buffer_ns": 3763166.6054949164, "calibration_cpu_ns": 5586875.0, "calibration_gpu_ticks": 5586875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3014038.0, "compute_span_ns": 3975500.0, "command_buffer_ns": 3792666.713707149, "calibration_cpu_ns": 5271000.0, "calibration_gpu_ticks": 5271000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3036746.0, "compute_span_ns": 4057625.0, "command_buffer_ns": 3878375.1078881323, "calibration_cpu_ns": 5208083.0, "calibration_gpu_ticks": 5208083.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 49417.0, "compute_span_ns": 49417.0, "command_buffer_ns": 49416.67430102825, "calibration_cpu_ns": 792916.0, "calibration_gpu_ticks": 792916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49625.0, "compute_span_ns": 49625.0, "command_buffer_ns": 49624.99951943755, "calibration_cpu_ns": 650458.0, "calibration_gpu_ticks": 650458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46250.0, "compute_span_ns": 46250.0, "command_buffer_ns": 46250.002924352884, "calibration_cpu_ns": 615667.0, "calibration_gpu_ticks": 615667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47791.0, "compute_span_ns": 47791.0, "command_buffer_ns": 47750.01434609294, "calibration_cpu_ns": 542166.0, "calibration_gpu_ticks": 542166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46959.0, "compute_span_ns": 46959.0, "command_buffer_ns": 47000.00863522291, "calibration_cpu_ns": 496125.0, "calibration_gpu_ticks": 496125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47875.0, "compute_span_ns": 47875.0, "command_buffer_ns": 47875.04440173507, "calibration_cpu_ns": 655792.0, "calibration_gpu_ticks": 655792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49833.0, "compute_span_ns": 49833.0, "command_buffer_ns": 49791.64805263281, "calibration_cpu_ns": 617583.0, "calibration_gpu_ticks": 617583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47875.0, "compute_span_ns": 47875.0, "command_buffer_ns": 47874.986194074154, "calibration_cpu_ns": 514083.0, "calibration_gpu_ticks": 514083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49167.0, "compute_span_ns": 49167.0, "command_buffer_ns": 49166.67239740491, "calibration_cpu_ns": 525792.0, "calibration_gpu_ticks": 525792.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 46.84765625, 47.482390625, 47.15621875, 46.64325, 47.05596875, 48.3685, 47.464265625, 47.09434375, 47.44915625 ], "compute_throughput_us_p50": 47.15621875, "compute_throughput_us_p90": 47.6596125, "command_buffer_throughput_us": [ 61.03645955590764, 60.59114639356267, 62.41341179702431, 62.59635392780183, 57.865233429765794, 61.7812502241577, 58.79947821085807, 59.260417401674204, 60.59961106075207 ], "command_buffer_throughput_us_p50": 60.59961106075207, "command_buffer_throughput_us_p90": 62.45000022317981, "compute_latency_us": [ 49.417, 49.625, 46.25, 47.791, 46.959, 47.875, 49.833, 47.875, 49.167 ], "compute_latency_us_p50": 47.875, "compute_latency_us_p90": 49.6666, "command_buffer_latency_us": [ 49.41667430102825, 49.62499951943755, 46.250002924352884, 47.75001434609294, 47.00000863522291, 47.87504440173507, 49.79164805263281, 47.874986194074154, 49.16667239740491 ], "command_buffer_latency_us_p50": 47.87504440173507, "command_buffer_latency_us_p90": 49.6583292260766, "counter_control_throughput_ratio": 2.0297904524151433, "counter_control_latency_ratio": 0.9203985228567987 }, "download_ms": 0.379792, "allocation_upload_ms": 0.587209, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-umdlrqkp/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7bac43b2781c01105a8e678fa6161555c0e89cbd2b0a98679fdc42bb1bd74b45", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.143959, "capture_ms": 0.075416, "compile_ms": 53.556459, "allocation_upload_ms": 4.087833, "cold_call_ms": 7.578791, "warmup_ms": 201.014459, "download_ms": 0.794333, "repetitions": 1226, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 22.5713026101, 24.5214787928, 22.2316476346, 24.8756460033, 22.5319127243, 24.9492936378, 22.4406264274, 24.2394306688, 22.4942903752 ], "latency_us": [ 352.833, 251.041, 256.792, 248.125, 224.625, 234.209, 333.75, 2125.666, 942.459 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 948208, "compute_span_ns": 948208, "calibration_cpu_ns": 3000542, "calibration_gpu_ticks": 3000542, "compute_passes": 1, "command_buffer_ns": 948291.679379, "command_buffers": 2 }, { "compute_ns": 944875, "compute_span_ns": 944875, "calibration_cpu_ns": 3102709, "calibration_gpu_ticks": 3102709, "compute_passes": 1, "command_buffer_ns": 944958.301261, "command_buffers": 2 }, { "compute_ns": 937250, "compute_span_ns": 937250, "calibration_cpu_ns": 2854292, "calibration_gpu_ticks": 2854292, "compute_passes": 1, "command_buffer_ns": 937333.38872, "command_buffers": 2 }, { "compute_ns": 1271875, "compute_span_ns": 1271875, "calibration_cpu_ns": 2905917, "calibration_gpu_ticks": 2905917, "compute_passes": 1, "command_buffer_ns": 1271958.34648, "command_buffers": 2 }, { "compute_ns": 1301041, "compute_span_ns": 1301041, "calibration_cpu_ns": 2727958, "calibration_gpu_ticks": 2727958, "compute_passes": 1, "command_buffer_ns": 1301041.68784, "command_buffers": 2 }, { "compute_ns": 922250, "compute_span_ns": 922250, "calibration_cpu_ns": 1392333, "calibration_gpu_ticks": 1392333, "compute_passes": 1, "command_buffer_ns": 922333.33271, "command_buffers": 2 }, { "compute_ns": 909625, "compute_span_ns": 909625, "calibration_cpu_ns": 3164500, "calibration_gpu_ticks": 3164500, "compute_passes": 1, "command_buffer_ns": 909750.000574, "command_buffers": 2 }, { "compute_ns": 1448375, "compute_span_ns": 1448375, "calibration_cpu_ns": 2973292, "calibration_gpu_ticks": 2973292, "compute_passes": 1, "command_buffer_ns": 1448458.35166, "command_buffers": 2 }, { "compute_ns": 928416, "compute_span_ns": 928416, "calibration_cpu_ns": 1464875, "calibration_gpu_ticks": 1464875, "compute_passes": 1, "command_buffer_ns": 928458.350245, "command_buffers": 2 } ], "latency": [ { "compute_ns": 16334, "compute_span_ns": 16334, "calibration_cpu_ns": 601792, "calibration_gpu_ticks": 601792, "compute_passes": 1, "command_buffer_ns": 16458.3907463, "command_buffers": 2 }, { "compute_ns": 51708, "compute_span_ns": 51708, "calibration_cpu_ns": 2133916, "calibration_gpu_ticks": 2133916, "compute_passes": 1, "command_buffer_ns": 51749.9283887, "command_buffers": 2 }, { "compute_ns": 24708, "compute_span_ns": 24708, "calibration_cpu_ns": 2540083, "calibration_gpu_ticks": 2540083, "compute_passes": 1, "command_buffer_ns": 24874.9856837, "command_buffers": 2 }, { "compute_ns": 16292, "compute_span_ns": 16292, "calibration_cpu_ns": 543625, "calibration_gpu_ticks": 543625, "compute_passes": 1, "command_buffer_ns": 16333.3606906, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 461542, "calibration_gpu_ticks": 461542, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "compute_ns": 28416, "compute_span_ns": 28416, "calibration_cpu_ns": 2371750, "calibration_gpu_ticks": 2371750, "compute_passes": 1, "command_buffer_ns": 28458.3074972, "command_buffers": 2 }, { "compute_ns": 32834, "compute_span_ns": 32834, "calibration_cpu_ns": 1552417, "calibration_gpu_ticks": 1552417, "compute_passes": 1, "command_buffer_ns": 32958.2835548, "command_buffers": 2 }, { "compute_ns": 19291, "compute_span_ns": 19291, "calibration_cpu_ns": 725833, "calibration_gpu_ticks": 725833, "compute_passes": 1, "command_buffer_ns": 19333.3835341, "command_buffers": 2 }, { "compute_ns": 20083, "compute_span_ns": 20083, "calibration_cpu_ns": 607125, "calibration_gpu_ticks": 607125, "compute_passes": 1, "command_buffer_ns": 20083.3310373, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 971833.302174, "command_buffers": 2 }, { "command_buffer_ns": 1407166.65192, "command_buffers": 2 }, { "command_buffer_ns": 928208.290134, "command_buffers": 2 }, { "command_buffer_ns": 1498208.32303, "command_buffers": 2 }, { "command_buffer_ns": 912208.342925, "command_buffers": 2 }, { "command_buffer_ns": 930791.66254, "command_buffers": 2 }, { "command_buffer_ns": 932333.350647, "command_buffers": 2 }, { "command_buffer_ns": 920833.379496, "command_buffers": 2 }, { "command_buffer_ns": 1290916.69805, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 23375.0324696, "command_buffers": 2 }, { "command_buffer_ns": 18000.0206456, "command_buffers": 2 }, { "command_buffer_ns": 18750.0263564, "command_buffers": 2 }, { "command_buffer_ns": 24166.7148657, "command_buffers": 2 }, { "command_buffer_ns": 16458.3325386, "command_buffers": 2 }, { "command_buffer_ns": 16208.330635, "command_buffers": 2 }, { "command_buffer_ns": 18708.4078789, "command_buffers": 2 }, { "command_buffer_ns": 18250.0225492, "command_buffers": 2 }, { "command_buffer_ns": 23124.9723583, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.18489534646875, 21.98697893625, 14.50325453334375, 23.40950504734375, 14.253255358203125, 14.543619727187501, 14.567708603859375, 14.388021554625, 20.17057340703125 ], "command_buffer_throughput_us_p50": 14.567708603859375, "command_buffer_throughput_us_p90": 22.271484158468752, "command_buffer_latency_us": [ 23.375032469599997, 18.000020645600003, 18.750026356400003, 24.1667148657, 16.4583325386, 16.208330635, 18.7084078789, 18.2500225492, 23.124972358300003 ], "command_buffer_latency_us_p50": 18.7084078789, "command_buffer_latency_us_p90": 23.533368948819998 }, "compute_throughput_us": [ 14.81575, 14.763671875, 14.64453125, 19.873046875, 20.328765625, 14.41015625, 14.212890625, 22.630859375, 14.5065 ], "compute_throughput_us_p50": 14.763671875, "compute_throughput_us_p90": 20.789184375, "command_buffer_throughput_us": [ 14.817057490296875, 14.764973457203125, 14.64583419875, 19.87434916375, 20.328776372500002, 14.41145832359375, 14.21484375896875, 22.632161744687497, 14.507161722578125 ], "command_buffer_throughput_us_p50": 14.764973457203125, "command_buffer_throughput_us_p90": 20.7894534469375, "compute_latency_us": [ 16.334, 51.708, 24.708, 16.292, 18.5, 28.416, 32.834, 19.291, 20.083 ], "compute_latency_us_p50": 20.083, "compute_latency_us_p90": 36.6088, "command_buffer_latency_us": [ 16.4583907463, 51.7499283887, 24.8749856837, 16.3333606906, 18.541642930400002, 28.4583074972, 32.9582835548, 19.3333835341, 20.0833310373 ], "command_buffer_latency_us_p50": 20.0833310373, "command_buffer_latency_us_p90": 36.716612521580004, "counter_control_throughput_ratio": 0.9757776013725904, "counter_control_latency_ratio": 1.1265808906773502 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 22.5713026101, "throughput_us_p90": 24.8903755302, "latency_us_p50": 256.792, "latency_us_p90": 1179.1004000000003 }, "slowdown": 0.5015609225441785, "device_compute_slowdown": 0.3130800616832748, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-027p_2sf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "5e168786762f89acc561194a669a3725da4fe77bed5800b1d573576d096bb3fc", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.600667, "capture_ms": 0.061167, "compile_ms": 55.937791, "allocation_upload_ms": 7.455958, "cold_call_ms": 9.51725, "warmup_ms": 200.720125, "download_ms": 3.194375, "repetitions": 311, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 103.428993569, 99.5602893891, 94.5379131833, 100.797694534, 102.509514469, 102.57529582, 94.5064308682, 100.633173633, 102.925376206 ], "latency_us": [ 366.167, 314.417, 333.708, 287.708, 353.792, 282.542, 371.291, 313.542, 2451.375 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4537958, "compute_span_ns": 4537958, "calibration_cpu_ns": 5230958, "calibration_gpu_ticks": 5230958, "compute_passes": 1, "command_buffer_ns": 4538041.65591, "command_buffers": 2 }, { "compute_ns": 4691084, "compute_span_ns": 4691084, "calibration_cpu_ns": 5338125, "calibration_gpu_ticks": 5338125, "compute_passes": 1, "command_buffer_ns": 4691125.01014, "command_buffers": 2 }, { "compute_ns": 4588416, "compute_span_ns": 4588416, "calibration_cpu_ns": 5241750, "calibration_gpu_ticks": 5241750, "compute_passes": 1, "command_buffer_ns": 4588458.33782, "command_buffers": 2 }, { "compute_ns": 4578917, "compute_span_ns": 4578917, "calibration_cpu_ns": 5324875, "calibration_gpu_ticks": 5324875, "compute_passes": 1, "command_buffer_ns": 4578958.26548, "command_buffers": 2 }, { "compute_ns": 4751334, "compute_span_ns": 4751334, "calibration_cpu_ns": 5360833, "calibration_gpu_ticks": 5360833, "compute_passes": 1, "command_buffer_ns": 4751500.03331, "command_buffers": 2 }, { "compute_ns": 4693542, "compute_span_ns": 4693542, "calibration_cpu_ns": 5366833, "calibration_gpu_ticks": 5366833, "compute_passes": 1, "command_buffer_ns": 4693624.97097, "command_buffers": 2 }, { "compute_ns": 4645833, "compute_span_ns": 4645833, "calibration_cpu_ns": 5389542, "calibration_gpu_ticks": 5389542, "compute_passes": 1, "command_buffer_ns": 4645833.33815, "command_buffers": 2 }, { "compute_ns": 4737542, "compute_span_ns": 4737542, "calibration_cpu_ns": 5391417, "calibration_gpu_ticks": 5391417, "compute_passes": 1, "command_buffer_ns": 4737583.33828, "command_buffers": 2 }, { "compute_ns": 4621708, "compute_span_ns": 4621708, "calibration_cpu_ns": 5751834, "calibration_gpu_ticks": 5751834, "compute_passes": 1, "command_buffer_ns": 4621708.24176, "command_buffers": 2 } ], "latency": [ { "compute_ns": 112458, "compute_span_ns": 112458, "calibration_cpu_ns": 816292, "calibration_gpu_ticks": 816292, "compute_passes": 1, "command_buffer_ns": 112499.983516, "command_buffers": 2 }, { "compute_ns": 81625, "compute_span_ns": 81625, "calibration_cpu_ns": 628666, "calibration_gpu_ticks": 628666, "compute_passes": 1, "command_buffer_ns": 81666.6870378, "command_buffers": 2 }, { "compute_ns": 111500, "compute_span_ns": 111500, "calibration_cpu_ns": 753209, "calibration_gpu_ticks": 753209, "compute_passes": 1, "command_buffer_ns": 111583.271064, "command_buffers": 2 }, { "compute_ns": 72916, "compute_span_ns": 72916, "calibration_cpu_ns": 562292, "calibration_gpu_ticks": 562292, "compute_passes": 1, "command_buffer_ns": 72916.6786186, "command_buffers": 2 }, { "compute_ns": 113041, "compute_span_ns": 113041, "calibration_cpu_ns": 712709, "calibration_gpu_ticks": 712709, "compute_passes": 1, "command_buffer_ns": 113000.04553, "command_buffers": 2 }, { "compute_ns": 72583, "compute_span_ns": 72583, "calibration_cpu_ns": 994250, "calibration_gpu_ticks": 994250, "compute_passes": 1, "command_buffer_ns": 72624.9418221, "command_buffers": 2 }, { "compute_ns": 71667, "compute_span_ns": 71667, "calibration_cpu_ns": 489292, "calibration_gpu_ticks": 489292, "compute_passes": 1, "command_buffer_ns": 71750.022471, "command_buffers": 2 }, { "compute_ns": 65333, "compute_span_ns": 65333, "calibration_cpu_ns": 535250, "calibration_gpu_ticks": 535250, "compute_passes": 1, "command_buffer_ns": 65333.3263472, "command_buffers": 2 }, { "compute_ns": 66291, "compute_span_ns": 66291, "calibration_cpu_ns": 495750, "calibration_gpu_ticks": 495750, "compute_passes": 1, "command_buffer_ns": 66375.0106469, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4427833.36621, "command_buffers": 2 }, { "command_buffer_ns": 4747500.00285, "command_buffers": 2 }, { "command_buffer_ns": 4471749.94042, "command_buffers": 2 }, { "command_buffer_ns": 4765416.67012, "command_buffers": 2 }, { "command_buffer_ns": 4740416.65439, "command_buffers": 2 }, { "command_buffer_ns": 4727750.02709, "command_buffers": 2 }, { "command_buffer_ns": 4662499.9959, "command_buffers": 2 }, { "command_buffer_ns": 4840249.95239, "command_buffers": 2 }, { "command_buffer_ns": 4812416.67876, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 403999.991249, "command_buffers": 2 }, { "command_buffer_ns": 64958.3525956, "command_buffers": 2 }, { "command_buffer_ns": 395291.659515, "command_buffers": 2 }, { "command_buffer_ns": 71833.3176337, "command_buffers": 2 }, { "command_buffer_ns": 485958.356876, "command_buffers": 2 }, { "command_buffer_ns": 375333.358534, "command_buffers": 2 }, { "command_buffer_ns": 112958.368845, "command_buffers": 2 }, { "command_buffer_ns": 482958.334032, "command_buffers": 2 }, { "command_buffer_ns": 111666.74085, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 69.18489634703124, 74.17968754453125, 69.8710928190625, 74.459635470625, 74.06901022484375, 73.87109417328125, 72.85156243593751, 75.62890550609376, 75.194010605625 ], "command_buffer_throughput_us_p50": 74.06901022484375, "command_buffer_throughput_us_p90": 75.28098958571874, "command_buffer_latency_us": [ 403.999991249, 64.9583525956, 395.291659515, 71.8333176337, 485.958356876, 375.333358534, 112.95836884500001, 482.958334032, 111.66674085 ], "command_buffer_latency_us_p50": 375.333358534, "command_buffer_latency_us_p90": 483.55833860079997 }, "compute_throughput_us": [ 70.90559375, 73.2981875, 71.694, 71.545578125, 74.23959375, 73.33659375, 72.591140625, 74.02409375, 72.2141875 ], "compute_throughput_us_p50": 72.591140625, "compute_throughput_us_p90": 74.06719375, "command_buffer_throughput_us": [ 70.90690087359376, 73.2988282834375, 71.6946615284375, 71.546222898125, 74.24218802046875, 73.33789017140626, 72.59114590859376, 74.024739660625, 72.21419127749999 ], "command_buffer_throughput_us_p50": 72.59114590859376, "command_buffer_throughput_us_p90": 74.06822933259375, "compute_latency_us": [ 112.458, 81.625, 111.5, 72.916, 113.041, 72.583, 71.667, 65.333, 66.291 ], "compute_latency_us_p50": 72.916, "compute_latency_us_p90": 112.5746, "command_buffer_latency_us": [ 112.49998351599999, 81.6666870378, 111.583271064, 72.9166786186, 113.00004553000001, 72.6249418221, 71.75002247100001, 65.3333263472, 66.37501064690001 ], "command_buffer_latency_us_p50": 72.9166786186, "command_buffer_latency_us_p90": 112.59999591879999, "counter_control_throughput_ratio": 0.9927819669135501, "counter_control_latency_ratio": 0.2822808637068291 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 100.797694534, "throughput_us_p90": 103.02609967859999, "latency_us_p50": 333.708, "latency_us_p90": 787.3078000000003 }, "torch": { "cold_call_ms": 7.813084, "warmup_ms": 200.718333, "repetitions": 155, "throughput_us": [ 178.78306451612903, 197.20053548387096, 181.29730967741935, 195.46370967741936, 179.32392903225806, 196.7766129032258, 182.31290322580645, 202.36209677419356, 194.56586451612904 ], "latency_us": [ 789.5, 892.75, 615.458, 635.459, 516.792, 409.791, 473.08299999999997, 566.875, 557.958 ], "throughput_us_p50": 194.56586451612904, "throughput_us_p90": 198.23284774193547, "latency_us_p50": 566.875, "latency_us_p90": 810.15, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 10643958.172295243, "command_buffers": 11 }, { "command_buffer_ns": 10704833.25747773, "command_buffers": 11 }, { "command_buffer_ns": 10681624.873541296, "command_buffers": 11 }, { "command_buffer_ns": 10623125.068377703, "command_buffers": 11 }, { "command_buffer_ns": 10833708.33657682, "command_buffers": 11 }, { "command_buffer_ns": 10583291.645161808, "command_buffers": 11 }, { "command_buffer_ns": 10453041.58469662, "command_buffers": 11 }, { "command_buffer_ns": 10734166.83357209, "command_buffers": 11 }, { "command_buffer_ns": 10530375.002417713, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 358874.9677874148, "command_buffers": 1 }, { "command_buffer_ns": 444624.9804459512, "command_buffers": 1 }, { "command_buffer_ns": 163833.31967517734, "command_buffers": 1 }, { "command_buffer_ns": 384208.3388008177, "command_buffers": 1 }, { "command_buffer_ns": 167416.69969633222, "command_buffers": 1 }, { "command_buffer_ns": 459208.32781121135, "command_buffers": 1 }, { "command_buffer_ns": 164500.0302232802, "command_buffers": 1 }, { "command_buffer_ns": 318999.9843016267, "command_buffers": 1 }, { "command_buffer_ns": 166083.33680778742, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 166.31184644211316, 167.26301964808954, 166.90038864908274, 165.98632919340162, 169.2766927590128, 165.36393195565324, 163.3287747608847, 167.7213567745639, 164.53710941277677 ], "command_buffer_throughput_us_p50": 166.31184644211316, "command_buffer_throughput_us_p90": 168.03242397145368, "command_buffer_latency_us": [ 358.8749677874148, 444.6249804459512, 163.83331967517734, 384.2083388008177, 167.41669969633222, 459.20832781121135, 164.5000302232802, 318.9999843016267, 166.08333680778742 ], "command_buffer_latency_us_p50": 318.9999843016267, "command_buffer_latency_us_p90": 447.54164991900325 }, "throughput": [ { "compute_ns": 11087335.0, "compute_span_ns": 12456541.0, "command_buffer_ns": 12211708.293762058, "calibration_cpu_ns": 13593208.0, "calibration_gpu_ticks": 13593208.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11008668.0, "compute_span_ns": 12329417.0, "command_buffer_ns": 12117208.272684366, "calibration_cpu_ns": 13492500.0, "calibration_gpu_ticks": 13492500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11150046.0, "compute_span_ns": 12511542.0, "command_buffer_ns": 12276749.941520393, "calibration_cpu_ns": 13853167.0, "calibration_gpu_ticks": 13853167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11032913.0, "compute_span_ns": 12361291.0, "command_buffer_ns": 12105625.122785568, "calibration_cpu_ns": 13508500.0, "calibration_gpu_ticks": 13508500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11396124.0, "compute_span_ns": 12566416.0, "command_buffer_ns": 12379333.551507443, "calibration_cpu_ns": 13378250.0, "calibration_gpu_ticks": 13378250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11194626.0, "compute_span_ns": 12650292.0, "command_buffer_ns": 12393374.985549599, "calibration_cpu_ns": 13650209.0, "calibration_gpu_ticks": 13650209.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11125329.0, "compute_span_ns": 12482125.0, "command_buffer_ns": 12237791.61227867, "calibration_cpu_ns": 13820333.0, "calibration_gpu_ticks": 13820333.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11027998.0, "compute_span_ns": 12399250.0, "command_buffer_ns": 12187458.45835656, "calibration_cpu_ns": 13660167.0, "calibration_gpu_ticks": 13660167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11119450.0, "compute_span_ns": 12524000.0, "command_buffer_ns": 12287874.88071248, "calibration_cpu_ns": 13863167.0, "calibration_gpu_ticks": 13863167.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 162916.0, "compute_span_ns": 162916.0, "command_buffer_ns": 162874.988745898, "calibration_cpu_ns": 754667.0, "calibration_gpu_ticks": 754667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 140333.0, "compute_span_ns": 140333.0, "command_buffer_ns": 140291.69688001275, "calibration_cpu_ns": 676292.0, "calibration_gpu_ticks": 676292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138667.0, "compute_span_ns": 138667.0, "command_buffer_ns": 138666.65540263057, "calibration_cpu_ns": 749042.0, "calibration_gpu_ticks": 749042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 139292.0, "compute_span_ns": 139292.0, "command_buffer_ns": 139291.68926551938, "calibration_cpu_ns": 721209.0, "calibration_gpu_ticks": 721209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135459.0, "compute_span_ns": 135459.0, "command_buffer_ns": 135500.0422336161, "calibration_cpu_ns": 601208.0, "calibration_gpu_ticks": 601208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138583.0, "compute_span_ns": 138583.0, "command_buffer_ns": 138583.30203220248, "calibration_cpu_ns": 634750.0, "calibration_gpu_ticks": 634750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 140042.0, "compute_span_ns": 140042.0, "command_buffer_ns": 140041.6367687285, "calibration_cpu_ns": 637166.0, "calibration_gpu_ticks": 637166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138583.0, "compute_span_ns": 138583.0, "command_buffer_ns": 138583.30203220248, "calibration_cpu_ns": 593791.0, "calibration_gpu_ticks": 593791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 153292.0, "compute_span_ns": 153292.0, "command_buffer_ns": 153333.35613831878, "calibration_cpu_ns": 941708.0, "calibration_gpu_ticks": 941708.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 173.239609375, 172.0104375, 174.21946875, 172.389265625, 178.0644375, 174.91603125, 173.833265625, 172.31246875, 173.74140625 ], "compute_throughput_us_p50": 173.74140625, "compute_throughput_us_p90": 175.5457125, "command_buffer_throughput_us": [ 190.80794209003216, 189.3313792606932, 191.82421783625614, 189.1503925435245, 193.4270867423038, 193.64648414921248, 191.21549394185422, 190.42903841182124, 191.9980450111325 ], "command_buffer_throughput_us_p50": 191.21549394185422, "command_buffer_throughput_us_p90": 193.47096622368554, "compute_latency_us": [ 162.916, 140.333, 138.667, 139.292, 135.459, 138.583, 140.042, 138.583, 153.292 ], "compute_latency_us_p50": 139.292, "compute_latency_us_p90": 155.2168, "command_buffer_latency_us": [ 162.874988745898, 140.29169688001275, 138.66665540263057, 139.29168926551938, 135.5000422336161, 138.58330203220248, 140.0416367687285, 138.58330203220248, 153.33335613831878 ], "command_buffer_latency_us_p50": 139.29168926551938, "command_buffer_latency_us_p90": 155.24168265983462, "counter_control_throughput_ratio": 1.1472901430172333, "counter_control_latency_ratio": 0.4538488425372136 }, "download_ms": 0.670959, "allocation_upload_ms": 17.333208, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.5180646398826251, "device_compute_slowdown": 0.41781140254239185, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.511583, "warmup_ms": 201.402667, "repetitions": 147, "throughput_us": [ 182.43395918367347, 201.14257142857144, 182.32880272108844, 181.0218231292517, 200.64880952380952, 179.9671156462585, 181.1408707482993, 187.4234693877551, 192.07369387755102 ], "latency_us": [ 629.084, 495.75, 437.625, 429.625, 434.041, 400.709, 370.916, 402.834, 696.2909999999999 ], "throughput_us_p50": 182.43395918367347, "throughput_us_p90": 200.7475619047619, "latency_us_p50": 434.041, "latency_us_p90": 642.5254, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 10499291.645828635, "command_buffers": 11 }, { "command_buffer_ns": 10429624.991957098, "command_buffers": 11 }, { "command_buffer_ns": 10967166.628688574, "command_buffers": 11 }, { "command_buffer_ns": 10525749.87988919, "command_buffers": 11 }, { "command_buffer_ns": 10595208.44137296, "command_buffers": 11 }, { "command_buffer_ns": 10343374.917283654, "command_buffers": 11 }, { "command_buffer_ns": 10357833.351008594, "command_buffers": 11 }, { "command_buffer_ns": 10696166.660636663, "command_buffers": 11 }, { "command_buffer_ns": 10444333.252962679, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 363000.0283010304, "command_buffers": 1 }, { "command_buffer_ns": 384166.6621156037, "command_buffers": 1 }, { "command_buffer_ns": 164500.0302232802, "command_buffers": 1 }, { "command_buffer_ns": 310041.7088717222, "command_buffers": 1 }, { "command_buffer_ns": 161625.0374354422, "command_buffers": 1 }, { "command_buffer_ns": 302625.00513345003, "command_buffers": 1 }, { "command_buffer_ns": 162708.28200504184, "command_buffers": 1 }, { "command_buffer_ns": 293374.9929070473, "command_buffers": 1 }, { "command_buffer_ns": 149125.000461936, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 164.0514319660724, 162.96289049932966, 171.36197857325897, 164.4648418732686, 165.5501318964525, 161.6152330825571, 161.84114610950928, 167.12760407244787, 163.19270707754185 ], "command_buffer_throughput_us_p50": 164.0514319660724, "command_buffer_throughput_us_p90": 167.9744789726101, "command_buffer_latency_us": [ 363.0000283010304, 384.1666621156037, 164.5000302232802, 310.0417088717222, 161.6250374354422, 302.62500513345003, 162.70828200504184, 293.3749929070473, 149.125000461936 ], "command_buffer_latency_us_p50": 293.3749929070473, "command_buffer_latency_us_p90": 367.23335506394506 }, "throughput": [ { "compute_ns": 11109002.0, "compute_span_ns": 12341625.0, "command_buffer_ns": 12149249.901995063, "calibration_cpu_ns": 13777334.0, "calibration_gpu_ticks": 13777334.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10834828.0, "compute_span_ns": 12120333.0, "command_buffer_ns": 11913416.674360633, "calibration_cpu_ns": 13958542.0, "calibration_gpu_ticks": 13958542.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10870754.0, "compute_span_ns": 12018167.0, "command_buffer_ns": 11843916.610814631, "calibration_cpu_ns": 13897083.0, "calibration_gpu_ticks": 13897083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10984244.0, "compute_span_ns": 12247250.0, "command_buffer_ns": 12042541.580740362, "calibration_cpu_ns": 13608041.0, "calibration_gpu_ticks": 13608041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10962084.0, "compute_span_ns": 12145750.0, "command_buffer_ns": 11973416.723776609, "calibration_cpu_ns": 13759000.0, "calibration_gpu_ticks": 13759000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10989082.0, "compute_span_ns": 12207125.0, "command_buffer_ns": 12018624.984193593, "calibration_cpu_ns": 13685791.0, "calibration_gpu_ticks": 13685791.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11032752.0, "compute_span_ns": 12246708.0, "command_buffer_ns": 12069791.671819985, "calibration_cpu_ns": 13757584.0, "calibration_gpu_ticks": 13757584.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 11092456.0, "compute_span_ns": 12498667.0, "command_buffer_ns": 12262124.97567758, "calibration_cpu_ns": 13839250.0, "calibration_gpu_ticks": 13839250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10905246.0, "compute_span_ns": 12344333.0, "command_buffer_ns": 12086500.006262213, "calibration_cpu_ns": 13575666.0, "calibration_gpu_ticks": 13575666.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 162625.0, "compute_span_ns": 162625.0, "command_buffer_ns": 162624.98684227467, "calibration_cpu_ns": 751958.0, "calibration_gpu_ticks": 751958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 139792.0, "compute_span_ns": 139792.0, "command_buffer_ns": 139833.3115503192, "calibration_cpu_ns": 707375.0, "calibration_gpu_ticks": 707375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138333.0, "compute_span_ns": 138333.0, "command_buffer_ns": 138333.35833624005, "calibration_cpu_ns": 613250.0, "calibration_gpu_ticks": 613250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136875.0, "compute_span_ns": 136875.0, "command_buffer_ns": 136875.02359971404, "calibration_cpu_ns": 647417.0, "calibration_gpu_ticks": 647417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137125.0, "compute_span_ns": 137125.0, "command_buffer_ns": 137124.96729567647, "calibration_cpu_ns": 693167.0, "calibration_gpu_ticks": 693167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137791.0, "compute_span_ns": 137791.0, "command_buffer_ns": 137750.00115856528, "calibration_cpu_ns": 611333.0, "calibration_gpu_ticks": 611333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138875.0, "compute_span_ns": 138875.0, "command_buffer_ns": 138874.98062103987, "calibration_cpu_ns": 719584.0, "calibration_gpu_ticks": 719584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 146292.0, "compute_span_ns": 146292.0, "command_buffer_ns": 146333.3028368652, "calibration_cpu_ns": 590541.0, "calibration_gpu_ticks": 590541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 144750.0, "compute_span_ns": 144750.0, "command_buffer_ns": 144749.99625235796, "calibration_cpu_ns": 644250.0, "calibration_gpu_ticks": 644250.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 173.57815625, 169.2941875, 169.85553125, 171.6288125, 171.2825625, 171.70440625, 172.38675, 173.319625, 170.39446875 ], "compute_throughput_us_p50": 171.6288125, "compute_throughput_us_p90": 173.37133125, "command_buffer_throughput_us": [ 189.83202971867286, 186.1471355368849, 185.0611970439786, 188.16471219906816, 187.0846363090095, 187.7910153780249, 188.59049487218726, 191.59570274496218, 188.85156259784708 ], "command_buffer_throughput_us_p50": 188.16471219906816, "command_buffer_throughput_us_p90": 190.18476432393072, "compute_latency_us": [ 162.625, 139.792, 138.333, 136.875, 137.125, 137.791, 138.875, 146.292, 144.75 ], "compute_latency_us_p50": 138.875, "compute_latency_us_p90": 149.5586, "command_buffer_latency_us": [ 162.62498684227467, 139.8333115503192, 138.33335833624005, 136.87502359971404, 137.12496729567647, 137.75000115856528, 138.87498062103987, 146.3333028368652, 144.74999625235796 ], "command_buffer_latency_us_p50": 138.87498062103987, "command_buffer_latency_us_p90": 149.59163963794708, "counter_control_throughput_ratio": 1.1464036944005234, "counter_control_latency_ratio": 0.49879269322463804 }, "download_ms": 0.644667, "allocation_upload_ms": 0.890041, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-2v6yqotm/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7f45b1ebfa78e24e3d477c2ad514fde5c49a3d7b63ed786757d9dc5d1979c8b1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.083208, "capture_ms": 0.07525, "compile_ms": 53.282542, "allocation_upload_ms": 5.515167, "cold_call_ms": 4.668416, "warmup_ms": 200.606625, "download_ms": 2.887875, "repetitions": 341, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 89.0986070381, 73.2339912023, 81.1180351906, 86.0825982405, 81.4710410557, 82.9557683284, 85.9490469208, 80.2178651026, 88.1317214076 ], "latency_us": [ 365.959, 292.375, 310.834, 270.042, 281.875, 295, 331.416, 306.583, 276.458 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3594541, "compute_span_ns": 3594541, "calibration_cpu_ns": 4376791, "calibration_gpu_ticks": 4376791, "compute_passes": 1, "command_buffer_ns": 3594541.57297, "command_buffers": 2 }, { "compute_ns": 4054750, "compute_span_ns": 4054750, "calibration_cpu_ns": 4972417, "calibration_gpu_ticks": 4972417, "compute_passes": 1, "command_buffer_ns": 4054791.70149, "command_buffers": 2 }, { "compute_ns": 4483166, "compute_span_ns": 4483166, "calibration_cpu_ns": 5454958, "calibration_gpu_ticks": 5454958, "compute_passes": 1, "command_buffer_ns": 4483166.67462, "command_buffers": 2 }, { "compute_ns": 4634958, "compute_span_ns": 4634958, "calibration_cpu_ns": 5552667, "calibration_gpu_ticks": 5552667, "compute_passes": 1, "command_buffer_ns": 4635083.3145, "command_buffers": 2 }, { "compute_ns": 4252333, "compute_span_ns": 4252333, "calibration_cpu_ns": 4950833, "calibration_gpu_ticks": 4950833, "compute_passes": 1, "command_buffer_ns": 4252416.6638, "command_buffers": 2 }, { "compute_ns": 4295417, "compute_span_ns": 4295417, "calibration_cpu_ns": 5185959, "calibration_gpu_ticks": 5185959, "compute_passes": 1, "command_buffer_ns": 4295458.37687, "command_buffers": 2 }, { "compute_ns": 4485667, "compute_span_ns": 4485667, "calibration_cpu_ns": 5352625, "calibration_gpu_ticks": 5352625, "compute_passes": 1, "command_buffer_ns": 4485791.6655, "command_buffers": 2 }, { "compute_ns": 4433583, "compute_span_ns": 4433583, "calibration_cpu_ns": 5380792, "calibration_gpu_ticks": 5380792, "compute_passes": 1, "command_buffer_ns": 4433666.70515, "command_buffers": 2 }, { "compute_ns": 4397292, "compute_span_ns": 4397292, "calibration_cpu_ns": 5200625, "calibration_gpu_ticks": 5200625, "compute_passes": 1, "command_buffer_ns": 4397333.3668, "command_buffers": 2 } ], "latency": [ { "compute_ns": 113083, "compute_span_ns": 113083, "calibration_cpu_ns": 916166, "calibration_gpu_ticks": 916166, "compute_passes": 1, "command_buffer_ns": 113083.340693, "command_buffers": 2 }, { "compute_ns": 62500, "compute_span_ns": 62500, "calibration_cpu_ns": 598084, "calibration_gpu_ticks": 598084, "compute_passes": 1, "command_buffer_ns": 62541.6869298, "command_buffers": 2 }, { "compute_ns": 58500, "compute_span_ns": 58500, "calibration_cpu_ns": 545958, "calibration_gpu_ticks": 545958, "compute_passes": 1, "command_buffer_ns": 58541.7146794, "command_buffers": 2 }, { "compute_ns": 58375, "compute_span_ns": 58375, "calibration_cpu_ns": 507167, "calibration_gpu_ticks": 507167, "compute_passes": 1, "command_buffer_ns": 58458.361309, "command_buffers": 2 }, { "compute_ns": 58292, "compute_span_ns": 58292, "calibration_cpu_ns": 526625, "calibration_gpu_ticks": 526625, "compute_passes": 1, "command_buffer_ns": 58375.0079386, "command_buffers": 2 }, { "compute_ns": 58208, "compute_span_ns": 58208, "calibration_cpu_ns": 511334, "calibration_gpu_ticks": 511334, "compute_passes": 1, "command_buffer_ns": 58250.0360906, "command_buffers": 2 }, { "compute_ns": 69708, "compute_span_ns": 69708, "calibration_cpu_ns": 464375, "calibration_gpu_ticks": 464375, "compute_passes": 1, "command_buffer_ns": 69666.6538715, "command_buffers": 2 }, { "compute_ns": 59583, "compute_span_ns": 59583, "calibration_cpu_ns": 469250, "calibration_gpu_ticks": 469250, "compute_passes": 1, "command_buffer_ns": 59625.0174567, "command_buffers": 2 }, { "compute_ns": 74584, "compute_span_ns": 74584, "calibration_cpu_ns": 555250, "calibration_gpu_ticks": 555250, "compute_passes": 1, "command_buffer_ns": 74666.691944, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 6433541.71189, "command_buffers": 2 }, { "command_buffer_ns": 4338500.03174, "command_buffers": 2 }, { "command_buffer_ns": 4239083.26775, "command_buffers": 2 }, { "command_buffer_ns": 4631416.63931, "command_buffers": 2 }, { "command_buffer_ns": 4432666.69754, "command_buffers": 2 }, { "command_buffer_ns": 4412833.36841, "command_buffers": 2 }, { "command_buffer_ns": 4244583.30963, "command_buffers": 2 }, { "command_buffer_ns": 4265249.99784, "command_buffers": 2 }, { "command_buffer_ns": 4390208.34165, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 377458.345611, "command_buffers": 2 }, { "command_buffer_ns": 468541.693408, "command_buffers": 2 }, { "command_buffer_ns": 107625.033706, "command_buffers": 2 }, { "command_buffer_ns": 557749.997824, "command_buffers": 2 }, { "command_buffer_ns": 105666.695163, "command_buffers": 2 }, { "command_buffer_ns": 377041.636966, "command_buffers": 2 }, { "command_buffer_ns": 109416.665509, "command_buffers": 2 }, { "command_buffer_ns": 419416.639488, "command_buffers": 2 }, { "command_buffer_ns": 109999.964479, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 100.52408924828124, 67.7890629959375, 66.23567605859374, 72.36588498921876, 69.2604171490625, 68.95052138140625, 66.32161421296875, 66.64453121625, 68.59700533828125 ], "command_buffer_throughput_us_p50": 68.59700533828125, "command_buffer_throughput_us_p90": 77.99752584103126, "command_buffer_latency_us": [ 377.458345611, 468.541693408, 107.625033706, 557.7499978239999, 105.666695163, 377.041636966, 109.416665509, 419.416639488, 109.999964479 ], "command_buffer_latency_us_p50": 377.041636966, "command_buffer_latency_us_p90": 486.3833542912 }, "compute_throughput_us": [ 56.164703125, 63.35546875, 70.04946875, 72.42121875, 66.442703125, 67.115890625, 70.088546875, 69.274734375, 68.7076875 ], "compute_throughput_us_p50": 68.7076875, "compute_throughput_us_p90": 70.55508125, "command_buffer_throughput_us": [ 56.164712077656255, 63.35612033578125, 70.0494792909375, 72.42317678906251, 66.44401037187501, 67.11653713859374, 70.0904947734375, 69.27604226796875, 68.70833385625 ], "command_buffer_throughput_us_p50": 68.70833385625, "command_buffer_throughput_us_p90": 70.5570311765625, "compute_latency_us": [ 113.083, 62.5, 58.5, 58.375, 58.292, 58.208, 69.708, 59.583, 74.584 ], "compute_latency_us_p50": 59.583, "compute_latency_us_p90": 82.28380000000001, "command_buffer_latency_us": [ 113.08334069300001, 62.5416869298, 58.5417146794, 58.458361309, 58.3750079386, 58.2500360906, 69.6666538715, 59.6250174567, 74.66669194400001 ], "command_buffer_latency_us_p50": 59.6250174567, "command_buffer_latency_us_p90": 82.35002169380002, "counter_control_throughput_ratio": 1.0007916962509653, "counter_control_latency_ratio": 0.29959157615113674 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 82.9557683284, "throughput_us_p90": 88.3250985337, "latency_us_p50": 295.0, "latency_us_p90": 338.32460000000003 }, "slowdown": 0.4547167024143822, "device_compute_slowdown": 0.4003272323520854, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-578_ayd5/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "8bb581805080afef9f47b3147742261e73f0bbc7f1d60afd69631b609a5e3f17", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 23.137, "capture_ms": 0.070417, "compile_ms": 55.560083, "allocation_upload_ms": 6.638916, "cold_call_ms": 6.103792, "warmup_ms": 201.429959, "download_ms": 2.339375, "repetitions": 519, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 75.9142581888, 71.2733622351, 77.0770712909, 75.8282755299, 71.2273603083, 77.1673082852, 78.9249344894, 75.5905587669, 75.5735375723 ], "latency_us": [ 1779.708, 453.75, 347.5, 281.75, 267.792, 287.416, 281.334, 323, 282.792 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4127417, "compute_span_ns": 4127417, "calibration_cpu_ns": 4884667, "calibration_gpu_ticks": 4884667, "compute_passes": 1, "command_buffer_ns": 4127499.99668, "command_buffers": 2 }, { "compute_ns": 4273458, "compute_span_ns": 4273458, "calibration_cpu_ns": 5012083, "calibration_gpu_ticks": 5012083, "compute_passes": 1, "command_buffer_ns": 4273458.32577, "command_buffers": 2 }, { "compute_ns": 4020458, "compute_span_ns": 4020458, "calibration_cpu_ns": 5099208, "calibration_gpu_ticks": 5099208, "compute_passes": 1, "command_buffer_ns": 4020458.37836, "command_buffers": 2 }, { "compute_ns": 3990792, "compute_span_ns": 3990792, "calibration_cpu_ns": 4797292, "calibration_gpu_ticks": 4797292, "compute_passes": 1, "command_buffer_ns": 3990791.67983, "command_buffers": 2 }, { "compute_ns": 3877208, "compute_span_ns": 3877208, "calibration_cpu_ns": 4807833, "calibration_gpu_ticks": 4807833, "compute_passes": 1, "command_buffer_ns": 3877208.33533, "command_buffers": 2 }, { "compute_ns": 3784292, "compute_span_ns": 3784292, "calibration_cpu_ns": 4878625, "calibration_gpu_ticks": 4878625, "compute_passes": 1, "command_buffer_ns": 3784374.9742, "command_buffers": 2 }, { "compute_ns": 3719625, "compute_span_ns": 3719625, "calibration_cpu_ns": 4453125, "calibration_gpu_ticks": 4453125, "compute_passes": 1, "command_buffer_ns": 3719666.68172, "command_buffers": 2 }, { "compute_ns": 3948500, "compute_span_ns": 3948500, "calibration_cpu_ns": 4858000, "calibration_gpu_ticks": 4858000, "compute_passes": 1, "command_buffer_ns": 3948541.64915, "command_buffers": 2 }, { "compute_ns": 3984625, "compute_span_ns": 3984625, "calibration_cpu_ns": 4939459, "calibration_gpu_ticks": 4939459, "compute_passes": 1, "command_buffer_ns": 3984666.66229, "command_buffers": 2 } ], "latency": [ { "compute_ns": 97750, "compute_span_ns": 97750, "calibration_cpu_ns": 633333, "calibration_gpu_ticks": 633333, "compute_passes": 1, "command_buffer_ns": 97874.9594651, "command_buffers": 2 }, { "compute_ns": 52417, "compute_span_ns": 52417, "calibration_cpu_ns": 565584, "calibration_gpu_ticks": 565584, "compute_passes": 1, "command_buffer_ns": 52458.3156221, "command_buffers": 2 }, { "compute_ns": 100166, "compute_span_ns": 100166, "calibration_cpu_ns": 920042, "calibration_gpu_ticks": 920042, "compute_passes": 1, "command_buffer_ns": 100208.329968, "command_buffers": 2 }, { "compute_ns": 52542, "compute_span_ns": 52542, "calibration_cpu_ns": 555250, "calibration_gpu_ticks": 555250, "compute_passes": 1, "command_buffer_ns": 52666.6408405, "command_buffers": 2 }, { "compute_ns": 98250, "compute_span_ns": 98250, "calibration_cpu_ns": 871833, "calibration_gpu_ticks": 871833, "compute_passes": 1, "command_buffer_ns": 98333.3447948, "command_buffers": 2 }, { "compute_ns": 59459, "compute_span_ns": 59459, "calibration_cpu_ns": 498375, "calibration_gpu_ticks": 498375, "compute_passes": 1, "command_buffer_ns": 59624.959249, "command_buffers": 2 }, { "compute_ns": 94750, "compute_span_ns": 94750, "calibration_cpu_ns": 718166, "calibration_gpu_ticks": 718166, "compute_passes": 1, "command_buffer_ns": 94875.053037, "command_buffers": 2 }, { "compute_ns": 52292, "compute_span_ns": 52292, "calibration_cpu_ns": 730500, "calibration_gpu_ticks": 730500, "compute_passes": 1, "command_buffer_ns": 52291.6670889, "command_buffers": 2 }, { "compute_ns": 95458, "compute_span_ns": 95458, "calibration_cpu_ns": 681125, "calibration_gpu_ticks": 681125, "compute_passes": 1, "command_buffer_ns": 95541.7053774, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3782916.69767, "command_buffers": 2 }, { "command_buffer_ns": 4260291.63646, "command_buffers": 2 }, { "command_buffer_ns": 3926833.33484, "command_buffers": 2 }, { "command_buffer_ns": 3952624.97477, "command_buffers": 2 }, { "command_buffer_ns": 3844666.644, "command_buffers": 2 }, { "command_buffer_ns": 3919791.66307, "command_buffers": 2 }, { "command_buffer_ns": 4098958.33582, "command_buffers": 2 }, { "command_buffer_ns": 3897208.3712, "command_buffers": 2 }, { "command_buffer_ns": 4013458.32506, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 447249.97133, "command_buffers": 2 }, { "command_buffer_ns": 53291.6747034, "command_buffers": 2 }, { "command_buffer_ns": 428791.68177, "command_buffers": 2 }, { "command_buffer_ns": 52874.9660589, "command_buffers": 2 }, { "command_buffer_ns": 437249.953393, "command_buffers": 2 }, { "command_buffer_ns": 52999.9379069, "command_buffers": 2 }, { "command_buffer_ns": 452958.338428, "command_buffers": 2 }, { "command_buffer_ns": 74958.3123252, "command_buffers": 2 }, { "command_buffer_ns": 427041.668445, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 59.108073401093755, 66.5670568196875, 61.356770856875, 61.759765230781255, 60.0729163125, 61.246744735468745, 64.0462239971875, 60.8938808, 62.710286329062505 ], "command_buffer_throughput_us_p50": 61.356770856875, "command_buffer_throughput_us_p90": 64.5503905616875, "command_buffer_latency_us": [ 447.24997133, 53.2916747034, 428.79168177, 52.8749660589, 437.249953393, 52.9999379069, 452.958338428, 74.9583123252, 427.041668445 ], "command_buffer_latency_us_p50": 427.041668445, "command_buffer_latency_us_p90": 448.3916447496 }, "compute_throughput_us": [ 64.490890625, 66.77278125, 62.81965625, 62.356125, 60.581375, 59.1295625, 58.119140625, 61.6953125, 62.259765625 ], "compute_throughput_us_p50": 62.259765625, "compute_throughput_us_p90": 64.94726875, "command_buffer_throughput_us": [ 64.492187448125, 66.77278634015624, 62.819662161875, 62.35611999734375, 60.58138023953125, 59.130858971875, 58.119791901875004, 61.69596326796875, 62.260416598281246 ], "command_buffer_throughput_us_p50": 62.260416598281246, "command_buffer_throughput_us_p90": 64.94830722653124, "compute_latency_us": [ 97.75, 52.417, 100.166, 52.542, 98.25, 59.459, 94.75, 52.292, 95.458 ], "compute_latency_us_p50": 94.75, "compute_latency_us_p90": 98.6332, "command_buffer_latency_us": [ 97.87495946509999, 52.4583156221, 100.208329968, 52.666640840499994, 98.3333447948, 59.624959249, 94.875053037, 52.2916670889, 95.54170537739999 ], "command_buffer_latency_us_p50": 94.875053037, "command_buffer_latency_us_p90": 98.70834182944, "counter_control_throughput_ratio": 1.0084641125858818, "counter_control_latency_ratio": 0.23369933286567543 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 75.8282755299, "throughput_us_p90": 77.51883352604, "latency_us_p50": 287.416, "latency_us_p90": 718.9416000000003 }, "torch": { "cold_call_ms": 4.503708, "warmup_ms": 200.498458, "repetitions": 171, "throughput_us": [ 178.14132748538012, 164.69834502923976, 174.73854970760235, 164.8830409356725, 174.4339649122807, 162.02631578947367, 177.70540935672514, 164.03557309941522, 180.18470175438597 ], "latency_us": [ 594.333, 427.66700000000003, 455.0, 427.29200000000003, 415.875, 406.709, 2177.25, 940.791, 468.833 ], "throughput_us_p50": 174.4339649122807, "throughput_us_p90": 178.55000233918128, "latency_us_p50": 455.0, "latency_us_p90": 1188.0828000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 9759583.335835487, "command_buffers": 11 }, { "command_buffer_ns": 9865875.006653368, "command_buffers": 11 }, { "command_buffer_ns": 9799958.323128521, "command_buffers": 11 }, { "command_buffer_ns": 9822583.349887282, "command_buffers": 11 }, { "command_buffer_ns": 9825000.015553087, "command_buffers": 11 }, { "command_buffer_ns": 9950666.630174965, "command_buffers": 11 }, { "command_buffer_ns": 9678916.539996862, "command_buffers": 11 }, { "command_buffer_ns": 10004083.334933966, "command_buffers": 11 }, { "command_buffer_ns": 9855833.370238543, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 351624.97078999877, "command_buffers": 1 }, { "command_buffer_ns": 138250.00496581197, "command_buffers": 1 }, { "command_buffer_ns": 248791.6499376297, "command_buffers": 1 }, { "command_buffer_ns": 130500.00416114926, "command_buffers": 1 }, { "command_buffer_ns": 362583.3196565509, "command_buffers": 1 }, { "command_buffer_ns": 128499.98893216252, "command_buffers": 1 }, { "command_buffer_ns": 315416.6624881327, "command_buffers": 1 }, { "command_buffer_ns": 130500.00416114926, "command_buffers": 1 }, { "command_buffer_ns": 365874.96288120747, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 152.49348962242948, 154.15429697895888, 153.12434879888315, 153.47786484198878, 153.51562524301698, 155.47916609648382, 151.23307093745098, 156.31380210834322, 153.99739640997723 ], "command_buffer_throughput_us_p50": 153.51562524301698, "command_buffer_throughput_us_p90": 155.6460932988557, "command_buffer_latency_us": [ 351.62497078999877, 138.25000496581197, 248.7916499376297, 130.50000416114926, 362.5833196565509, 128.49998893216252, 315.4166624881327, 130.50000416114926, 365.87496288120747 ], "command_buffer_latency_us_p50": 248.7916499376297, "command_buffer_latency_us_p90": 363.2416483014822 }, "throughput": [ { "compute_ns": 10256254.0, "compute_span_ns": 11465167.0, "command_buffer_ns": 11301041.638944298, "calibration_cpu_ns": 12865625.0, "calibration_gpu_ticks": 12865625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10147580.0, "compute_span_ns": 11370167.0, "command_buffer_ns": 11192875.041160733, "calibration_cpu_ns": 12723959.0, "calibration_gpu_ticks": 12723959.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10126750.0, "compute_span_ns": 11404000.0, "command_buffer_ns": 11204249.982256442, "calibration_cpu_ns": 12513125.0, "calibration_gpu_ticks": 12513125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10224954.0, "compute_span_ns": 11461834.0, "command_buffer_ns": 11284208.44906941, "calibration_cpu_ns": 12681500.0, "calibration_gpu_ticks": 12681500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10247581.0, "compute_span_ns": 11449541.0, "command_buffer_ns": 11261958.279646933, "calibration_cpu_ns": 12830834.0, "calibration_gpu_ticks": 12830834.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10241206.0, "compute_span_ns": 11564667.0, "command_buffer_ns": 11357333.161868155, "calibration_cpu_ns": 12823416.0, "calibration_gpu_ticks": 12823416.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10161667.0, "compute_span_ns": 11463625.0, "command_buffer_ns": 11283374.97357279, "calibration_cpu_ns": 12761083.0, "calibration_gpu_ticks": 12761083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10241751.0, "compute_span_ns": 11500625.0, "command_buffer_ns": 11294041.702058166, "calibration_cpu_ns": 13015708.0, "calibration_gpu_ticks": 13015708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10422704.0, "compute_span_ns": 11600084.0, "command_buffer_ns": 11408749.90960583, "calibration_cpu_ns": 13665875.0, "calibration_gpu_ticks": 13665875.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 144542.0, "compute_span_ns": 144542.0, "command_buffer_ns": 144583.3477191627, "calibration_cpu_ns": 709250.0, "calibration_gpu_ticks": 709250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 121875.0, "compute_span_ns": 121875.0, "command_buffer_ns": 121874.9675899744, "calibration_cpu_ns": 672625.0, "calibration_gpu_ticks": 672625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 141667.0, "compute_span_ns": 141667.0, "command_buffer_ns": 141666.67824611068, "calibration_cpu_ns": 617750.0, "calibration_gpu_ticks": 617750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136125.0, "compute_span_ns": 136125.0, "command_buffer_ns": 136125.017888844, "calibration_cpu_ns": 608875.0, "calibration_gpu_ticks": 608875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 148250.0, "compute_span_ns": 148250.0, "command_buffer_ns": 148250.02290308475, "calibration_cpu_ns": 803584.0, "calibration_gpu_ticks": 803584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125584.0, "compute_span_ns": 125584.0, "command_buffer_ns": 125624.99614432454, "calibration_cpu_ns": 658958.0, "calibration_gpu_ticks": 658958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 150375.0, "compute_span_ns": 150375.0, "command_buffer_ns": 150375.0099800527, "calibration_cpu_ns": 792125.0, "calibration_gpu_ticks": 792125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 128292.0, "compute_span_ns": 128292.0, "command_buffer_ns": 128291.66371375322, "calibration_cpu_ns": 618667.0, "calibration_gpu_ticks": 618667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 143250.0, "compute_span_ns": 143250.0, "command_buffer_ns": 143250.04303827882, "calibration_cpu_ns": 794084.0, "calibration_gpu_ticks": 794084.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 160.25396875, 158.5559375, 158.23046875, 159.76490625, 160.118453125, 160.01884375, 158.776046875, 160.027359375, 162.85475 ], "compute_throughput_us_p50": 160.01884375, "compute_throughput_us_p90": 160.774125, "command_buffer_throughput_us": [ 176.57877560850466, 174.88867251813645, 175.0664059727569, 176.31575701670954, 175.96809811948333, 177.45833065418992, 176.30273396207485, 176.46940159465885, 178.2617173375911 ], "command_buffer_throughput_us_p50": 176.31575701670954, "command_buffer_throughput_us_p90": 177.61900799087016, "compute_latency_us": [ 144.542, 121.875, 141.667, 136.125, 148.25, 125.584, 150.375, 128.292, 143.25 ], "compute_latency_us_p50": 141.667, "compute_latency_us_p90": 148.675, "command_buffer_latency_us": [ 144.5833477191627, 121.8749675899744, 141.66667824611068, 136.125017888844, 148.25002290308475, 125.62499614432454, 150.3750099800527, 128.29166371375322, 143.25004303827882 ], "command_buffer_latency_us_p50": 141.66667824611068, "command_buffer_latency_us_p90": 148.67502031847835, "counter_control_throughput_ratio": 1.1462552938238295, "counter_control_latency_ratio": 0.5694189426438769 }, "download_ms": 0.606875, "allocation_upload_ms": 16.81025, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.4347104967087832, "device_compute_slowdown": 0.38907771213663706, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.574875, "warmup_ms": 200.787625, "repetitions": 160, "throughput_us": [ 164.17890625, 172.68515625, 171.10520625, 167.38229375, 168.1645875, 182.10495, 168.59791875000002, 167.31744375, 175.29635624999997 ], "latency_us": [ 1035.875, 596.4580000000001, 415.334, 417.541, 398.208, 391.875, 413.58299999999997, 402.91700000000003, 382.875 ], "throughput_us_p50": 168.59791875000002, "throughput_us_p90": 176.65807499999997, "latency_us_p50": 413.58299999999997, "latency_us_p90": 684.3414000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 10274250.0603199, "command_buffers": 11 }, { "command_buffer_ns": 9964666.56215489, "command_buffers": 11 }, { "command_buffer_ns": 9737333.341035992, "command_buffers": 11 }, { "command_buffer_ns": 9991041.617468, "command_buffers": 11 }, { "command_buffer_ns": 9760958.258993924, "command_buffers": 11 }, { "command_buffer_ns": 9996458.364184946, "command_buffers": 11 }, { "command_buffer_ns": 9775624.78510663, "command_buffers": 11 }, { "command_buffer_ns": 10133375.006262213, "command_buffers": 11 }, { "command_buffer_ns": 9727166.674565524, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 472000.04337355494, "command_buffers": 1 }, { "command_buffer_ns": 125500.02429634333, "command_buffers": 1 }, { "command_buffer_ns": 267874.9733604491, "command_buffers": 1 }, { "command_buffer_ns": 252874.97555837035, "command_buffers": 1 }, { "command_buffer_ns": 149583.32758396864, "command_buffers": 1 }, { "command_buffer_ns": 142125.00536814332, "command_buffers": 1 }, { "command_buffer_ns": 132916.6698269546, "command_buffers": 1 }, { "command_buffer_ns": 125874.99804794788, "command_buffers": 1 }, { "command_buffer_ns": 128583.34230259061, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 160.53515719249845, 155.69791503367014, 152.14583345368737, 156.1100252729375, 152.51497279678006, 156.19466194038978, 152.74413726729108, 158.33398447284708, 151.9869792900863 ], "command_buffer_throughput_us_p50": 155.69791503367014, "command_buffer_throughput_us_p90": 158.77421901677735, "command_buffer_latency_us": [ 472.00004337355494, 125.50002429634333, 267.8749733604491, 252.87497555837035, 149.58332758396864, 142.12500536814332, 132.9166698269546, 125.87499804794788, 128.5833423025906 ], "command_buffer_latency_us_p50": 142.12500536814332, "command_buffer_latency_us_p90": 308.6999873630703 }, "throughput": [ { "compute_ns": 10573834.0, "compute_span_ns": 11789584.0, "command_buffer_ns": 11583791.754674166, "calibration_cpu_ns": 13445792.0, "calibration_gpu_ticks": 13445792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10185086.0, "compute_span_ns": 11393542.0, "command_buffer_ns": 11183041.671756655, "calibration_cpu_ns": 12922959.0, "calibration_gpu_ticks": 12922959.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10104210.0, "compute_span_ns": 11390166.0, "command_buffer_ns": 11195166.676770896, "calibration_cpu_ns": 12861208.0, "calibration_gpu_ticks": 12861208.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10442042.0, "compute_span_ns": 11696125.0, "command_buffer_ns": 11489708.151202649, "calibration_cpu_ns": 13002583.0, "calibration_gpu_ticks": 13002583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10531841.0, "compute_span_ns": 11722291.0, "command_buffer_ns": 11518458.370119333, "calibration_cpu_ns": 13018625.0, "calibration_gpu_ticks": 13018625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10241375.0, "compute_span_ns": 11471708.0, "command_buffer_ns": 11260583.240073174, "calibration_cpu_ns": 12839208.0, "calibration_gpu_ticks": 12839208.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10254543.0, "compute_span_ns": 11516417.0, "command_buffer_ns": 11316833.31914246, "calibration_cpu_ns": 12895083.0, "calibration_gpu_ticks": 12895083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10248624.0, "compute_span_ns": 11591542.0, "command_buffer_ns": 11353666.719514877, "calibration_cpu_ns": 13303375.0, "calibration_gpu_ticks": 13303375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 10300090.0, "compute_span_ns": 11492709.0, "command_buffer_ns": 11293166.724499315, "calibration_cpu_ns": 12475833.0, "calibration_gpu_ticks": 12475833.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 151583.0, "compute_span_ns": 151583.0, "command_buffer_ns": 151541.66612774134, "calibration_cpu_ns": 808333.0, "calibration_gpu_ticks": 808333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 131417.0, "compute_span_ns": 131417.0, "command_buffer_ns": 131416.65840521455, "calibration_cpu_ns": 725916.0, "calibration_gpu_ticks": 725916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 141583.0, "compute_span_ns": 141583.0, "command_buffer_ns": 141541.64819046855, "calibration_cpu_ns": 634291.0, "calibration_gpu_ticks": 634291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 126209.0, "compute_span_ns": 126209.0, "command_buffer_ns": 126250.03000721335, "calibration_cpu_ns": 723500.0, "calibration_gpu_ticks": 723500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125750.0, "compute_span_ns": 125750.0, "command_buffer_ns": 125750.02619996667, "calibration_cpu_ns": 696917.0, "calibration_gpu_ticks": 696917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 274583.0, "compute_span_ns": 274583.0, "command_buffer_ns": 274541.67138785124, "calibration_cpu_ns": 1527000.0, "calibration_gpu_ticks": 1527000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 399291.0, "compute_span_ns": 399291.0, "command_buffer_ns": 399250.0132881105, "calibration_cpu_ns": 1413792.0, "calibration_gpu_ticks": 1413792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 149792.0, "compute_span_ns": 149792.0, "command_buffer_ns": 149791.71101003885, "calibration_cpu_ns": 843375.0, "calibration_gpu_ticks": 843375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 343250.0, "compute_span_ns": 343250.0, "command_buffer_ns": 343249.99433010817, "calibration_cpu_ns": 1761250.0, "calibration_gpu_ticks": 1761250.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 165.21615625, 159.14196875, 157.87828125, 163.15690625, 164.560015625, 160.021484375, 160.227234375, 160.13475, 160.93890625 ], "compute_throughput_us_p50": 160.227234375, "compute_throughput_us_p90": 164.69124375, "command_buffer_throughput_us": [ 180.99674616678385, 174.73502612119773, 174.92447932454525, 179.5266898625414, 179.97591203311458, 175.94661312614335, 176.82552061160095, 177.40104249241995, 176.4557300703018 ], "command_buffer_throughput_us_p50": 176.82552061160095, "command_buffer_throughput_us_p90": 180.18007885984844, "compute_latency_us": [ 151.583, 131.417, 141.583, 126.209, 125.75, 274.583, 399.291, 149.792, 343.25 ], "compute_latency_us_p50": 149.792, "compute_latency_us_p90": 354.45820000000003, "command_buffer_latency_us": [ 151.54166612774134, 131.41665840521455, 141.54164819046855, 126.25003000721335, 125.75002619996667, 274.54167138785124, 399.2500132881105, 149.79171101003885, 343.24999433010817 ], "command_buffer_latency_us_p50": 149.79171101003885, "command_buffer_latency_us_p90": 354.44999812170863, "counter_control_throughput_ratio": 1.1497158703185362, "counter_control_latency_ratio": 1.0471444857644032 }, "download_ms": 0.614, "allocation_upload_ms": 0.806, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-89jka_sc/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "117627a7b60c4cee3b9bce9542bc5f280f025f1ad809d52b8d22cb3cdd24d0d1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.886917, "capture_ms": 0.066167, "compile_ms": 53.942667, "allocation_upload_ms": 6.339333, "cold_call_ms": 9.40875, "warmup_ms": 200.130208, "download_ms": 2.806625, "repetitions": 364, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 83.6239697802, 75.1686126374, 82.6092032967, 75.1127527473, 76.96875, 80.8721373626, 75.1097747253, 83.0869945055, 75.3241758242 ], "latency_us": [ 362.416, 353, 1882.708, 656.417, 392.25, 312.917, 280.541, 297.584, 279.375 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4130958, "compute_span_ns": 4130958, "calibration_cpu_ns": 5049000, "calibration_gpu_ticks": 5049000, "compute_passes": 1, "command_buffer_ns": 4131041.70002, "command_buffers": 2 }, { "compute_ns": 4092042, "compute_span_ns": 4092042, "calibration_cpu_ns": 5010041, "calibration_gpu_ticks": 5010041, "compute_passes": 1, "command_buffer_ns": 4092166.66594, "command_buffers": 2 }, { "compute_ns": 4044375, "compute_span_ns": 4044375, "calibration_cpu_ns": 5036167, "calibration_gpu_ticks": 5036167, "compute_passes": 1, "command_buffer_ns": 4044416.6516, "command_buffers": 2 }, { "compute_ns": 4114583, "compute_span_ns": 4114583, "calibration_cpu_ns": 5068458, "calibration_gpu_ticks": 5068458, "compute_passes": 1, "command_buffer_ns": 4114708.33933, "command_buffers": 2 }, { "compute_ns": 4090000, "compute_span_ns": 4090000, "calibration_cpu_ns": 5095875, "calibration_gpu_ticks": 5095875, "compute_passes": 1, "command_buffer_ns": 4090083.35555, "command_buffers": 2 }, { "compute_ns": 4083500, "compute_span_ns": 4083500, "calibration_cpu_ns": 5080708, "calibration_gpu_ticks": 5080708, "compute_passes": 1, "command_buffer_ns": 4083500.01089, "command_buffers": 2 }, { "compute_ns": 4180083, "compute_span_ns": 4180083, "calibration_cpu_ns": 5174292, "calibration_gpu_ticks": 5174292, "compute_passes": 1, "command_buffer_ns": 4180083.34236, "command_buffers": 2 }, { "compute_ns": 4004209, "compute_span_ns": 4004209, "calibration_cpu_ns": 4973250, "calibration_gpu_ticks": 4973250, "compute_passes": 1, "command_buffer_ns": 4004375.01958, "command_buffers": 2 }, { "compute_ns": 4119167, "compute_span_ns": 4119167, "calibration_cpu_ns": 4953208, "calibration_gpu_ticks": 4953208, "compute_passes": 1, "command_buffer_ns": 4119291.66876, "command_buffers": 2 } ], "latency": [ { "compute_ns": 96334, "compute_span_ns": 96334, "calibration_cpu_ns": 721541, "calibration_gpu_ticks": 721541, "compute_passes": 1, "command_buffer_ns": 96458.3014138, "command_buffers": 2 }, { "compute_ns": 54541, "compute_span_ns": 54541, "calibration_cpu_ns": 466542, "calibration_gpu_ticks": 466542, "compute_passes": 1, "command_buffer_ns": 54583.3609067, "command_buffers": 2 }, { "compute_ns": 98750, "compute_span_ns": 98750, "calibration_cpu_ns": 694625, "calibration_gpu_ticks": 694625, "compute_passes": 1, "command_buffer_ns": 98874.9670796, "command_buffers": 2 }, { "compute_ns": 54833, "compute_span_ns": 54833, "calibration_cpu_ns": 483583, "calibration_gpu_ticks": 483583, "compute_passes": 1, "command_buffer_ns": 54958.2764506, "command_buffers": 2 }, { "compute_ns": 318917, "compute_span_ns": 318917, "calibration_cpu_ns": 2041958, "calibration_gpu_ticks": 2041958, "compute_passes": 1, "command_buffer_ns": 318958.365824, "command_buffers": 2 }, { "compute_ns": 91666, "compute_span_ns": 91666, "calibration_cpu_ns": 602875, "calibration_gpu_ticks": 602875, "compute_passes": 1, "command_buffer_ns": 91708.265245, "command_buffers": 2 }, { "compute_ns": 184833, "compute_span_ns": 184833, "calibration_cpu_ns": 1924208, "calibration_gpu_ticks": 1924208, "compute_passes": 1, "command_buffer_ns": 184791.628271, "command_buffers": 2 }, { "compute_ns": 62791, "compute_span_ns": 62791, "calibration_cpu_ns": 489709, "calibration_gpu_ticks": 489709, "compute_passes": 1, "command_buffer_ns": 62791.6888334, "command_buffers": 2 }, { "compute_ns": 369333, "compute_span_ns": 369333, "calibration_cpu_ns": 1651167, "calibration_gpu_ticks": 1651167, "compute_passes": 1, "command_buffer_ns": 369333.254639, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4075583.30335, "command_buffers": 2 }, { "command_buffer_ns": 4128499.94609, "command_buffers": 2 }, { "command_buffer_ns": 3977124.9867, "command_buffers": 2 }, { "command_buffer_ns": 4025333.38638, "command_buffers": 2 }, { "command_buffer_ns": 3906874.97566, "command_buffers": 2 }, { "command_buffer_ns": 4169041.64013, "command_buffers": 2 }, { "command_buffer_ns": 4084000.0147, "command_buffers": 2 }, { "command_buffer_ns": 3828666.63858, "command_buffers": 2 }, { "command_buffer_ns": 4174500.06353, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 427166.640293, "command_buffers": 2 }, { "command_buffer_ns": 54708.3909623, "command_buffers": 2 }, { "command_buffer_ns": 476874.993183, "command_buffers": 2 }, { "command_buffer_ns": 54708.3327547, "command_buffers": 2 }, { "command_buffer_ns": 54124.975577, "command_buffers": 2 }, { "command_buffer_ns": 54458.330851, "command_buffers": 2 }, { "command_buffer_ns": 57208.3517909, "command_buffers": 2 }, { "command_buffer_ns": 54750.0094399, "command_buffers": 2 }, { "command_buffer_ns": 54375.0356883, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 63.680989114843754, 64.50781165765625, 62.142577917187495, 62.8958341621875, 61.0449214946875, 65.14127562703125, 63.8125002296875, 59.8229162278125, 65.22656349265625 ], "command_buffer_throughput_us_p50": 63.680989114843754, "command_buffer_throughput_us_p90": 65.15833320015625, "command_buffer_latency_us": [ 427.166640293, 54.7083909623, 476.874993183, 54.7083327547, 54.124975577, 54.458330851, 57.208351790900004, 54.7500094399, 54.375035688299995 ], "command_buffer_latency_us_p50": 54.7083909623, "command_buffer_latency_us_p90": 437.108310871 }, "compute_throughput_us": [ 64.54621875, 63.93815625, 63.193359375, 64.290359375, 63.90625, 63.8046875, 65.313796875, 62.565765625, 64.361984375 ], "compute_throughput_us_p50": 63.93815625, "compute_throughput_us_p90": 64.69973437499999, "command_buffer_throughput_us": [ 64.5475265628125, 63.940104155312504, 63.19401018125, 64.29231780203125, 63.90755243046875, 63.80468767015625, 65.313802224375, 62.568359680937505, 64.363932324375 ], "command_buffer_throughput_us_p50": 63.940104155312504, "command_buffer_throughput_us_p90": 64.70078169512499, "compute_latency_us": [ 96.334, 54.541, 98.75, 54.833, 318.917, 91.666, 184.833, 62.791, 369.333 ], "compute_latency_us_p50": 96.334, "compute_latency_us_p90": 329.0002, "command_buffer_latency_us": [ 96.4583014138, 54.5833609067, 98.8749670796, 54.9582764506, 318.958365824, 91.708265245, 184.791628271, 62.7916888334, 369.333254639 ], "command_buffer_latency_us_p50": 96.4583014138, "command_buffer_latency_us_p90": 329.033343587, "counter_control_throughput_ratio": 1.016919675676533, "counter_control_latency_ratio": 1.146879963597586 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 76.96875, "throughput_us_p90": 83.19438956044, "latency_us_p50": 353.0, "latency_us_p90": 901.6752000000002 }, "slowdown": 0.4565225393685353, "device_compute_slowdown": 0.3990467444526782, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-32p7fmdu/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "e81310ef6c7b3435af776cabf332187d02f0ddc801b9234c32af8a56116f248b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 23.083834, "capture_ms": 0.060792, "compile_ms": 65.397875, "allocation_upload_ms": 1.349708, "cold_call_ms": 2.85975, "warmup_ms": 200.239833, "download_ms": 0.343709, "repetitions": 3802, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.32251657023, 5.8110973172, 5.92162019989, 6.3086094161, 6.31095476065, 6.31872488164, 6.002783798, 5.75233429774, 6.31080142031 ], "latency_us": [ 248.792, 212.792, 194.958, 187.25, 178.416, 183.583, 206.209, 209.625, 1288.25 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 48, "normalized_kernel_cost": 48, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 298500, "compute_span_ns": 298500, "calibration_cpu_ns": 892375, "calibration_gpu_ticks": 892375, "compute_passes": 1, "command_buffer_ns": 298666.651361, "command_buffers": 2 }, { "compute_ns": 295042, "compute_span_ns": 295042, "calibration_cpu_ns": 751333, "calibration_gpu_ticks": 751333, "compute_passes": 1, "command_buffer_ns": 295125.006232, "command_buffers": 2 }, { "compute_ns": 943167, "compute_span_ns": 943167, "calibration_cpu_ns": 1892792, "calibration_gpu_ticks": 1892792, "compute_passes": 1, "command_buffer_ns": 943250.022829, "command_buffers": 2 }, { "compute_ns": 307792, "compute_span_ns": 307792, "calibration_cpu_ns": 728625, "calibration_gpu_ticks": 728625, "compute_passes": 1, "command_buffer_ns": 307833.368424, "command_buffers": 2 }, { "compute_ns": 980417, "compute_span_ns": 980417, "calibration_cpu_ns": 2041375, "calibration_gpu_ticks": 2041375, "compute_passes": 1, "command_buffer_ns": 980583.310593, "command_buffers": 2 }, { "compute_ns": 276042, "compute_span_ns": 276042, "calibration_cpu_ns": 710833, "calibration_gpu_ticks": 710833, "compute_passes": 1, "command_buffer_ns": 276208.273135, "command_buffers": 2 }, { "compute_ns": 307458, "compute_span_ns": 307458, "calibration_cpu_ns": 761958, "calibration_gpu_ticks": 761958, "compute_passes": 1, "command_buffer_ns": 307583.308313, "command_buffers": 2 }, { "compute_ns": 283250, "compute_span_ns": 283250, "calibration_cpu_ns": 735542, "calibration_gpu_ticks": 735542, "compute_passes": 1, "command_buffer_ns": 283333.298285, "command_buffers": 2 }, { "compute_ns": 305208, "compute_span_ns": 305208, "calibration_cpu_ns": 752792, "calibration_gpu_ticks": 752792, "compute_passes": 1, "command_buffer_ns": 305208.37754, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6541, "compute_span_ns": 6541, "calibration_cpu_ns": 473667, "calibration_gpu_ticks": 473667, "compute_passes": 1, "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "compute_ns": 31250, "compute_span_ns": 31250, "calibration_cpu_ns": 951375, "calibration_gpu_ticks": 951375, "compute_passes": 1, "command_buffer_ns": 31333.3584927, "command_buffers": 2 }, { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 372000, "calibration_gpu_ticks": 372000, "compute_passes": 1, "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 362708, "calibration_gpu_ticks": 362708, "compute_passes": 1, "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "compute_ns": 40917, "compute_span_ns": 40917, "calibration_cpu_ns": 509375, "calibration_gpu_ticks": 509375, "compute_passes": 1, "command_buffer_ns": 40958.3444707, "command_buffers": 2 }, { "compute_ns": 7334, "compute_span_ns": 7334, "calibration_cpu_ns": 426708, "calibration_gpu_ticks": 426708, "compute_passes": 1, "command_buffer_ns": 7416.70373827, "command_buffers": 2 }, { "compute_ns": 8959, "compute_span_ns": 8959, "calibration_cpu_ns": 511000, "calibration_gpu_ticks": 511000, "compute_passes": 1, "command_buffer_ns": 9083.30548555, "command_buffers": 2 }, { "compute_ns": 228041, "compute_span_ns": 228041, "calibration_cpu_ns": 1607375, "calibration_gpu_ticks": 1607375, "compute_passes": 1, "command_buffer_ns": 228041.724768, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 344500, "calibration_gpu_ticks": 344500, "compute_passes": 1, "command_buffer_ns": 6583.28644931, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 684250.029735, "command_buffers": 2 }, { "command_buffer_ns": 276208.331343, "command_buffers": 2 }, { "command_buffer_ns": 263083.318714, "command_buffers": 2 }, { "command_buffer_ns": 270916.614681, "command_buffers": 2 }, { "command_buffer_ns": 261708.337348, "command_buffers": 2 }, { "command_buffer_ns": 307583.308313, "command_buffers": 2 }, { "command_buffer_ns": 905583.321583, "command_buffers": 2 }, { "command_buffer_ns": 262000.015937, "command_buffers": 2 }, { "command_buffer_ns": 975541.712251, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6624.96313453, "command_buffers": 2 }, { "command_buffer_ns": 31083.3565891, "command_buffers": 2 }, { "command_buffer_ns": 7208.37851986, "command_buffers": 2 }, { "command_buffer_ns": 10333.3732113, "command_buffers": 2 }, { "command_buffer_ns": 7750.00080466, "command_buffers": 2 }, { "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "command_buffer_ns": 6916.69993103, "command_buffers": 2 }, { "command_buffer_ns": 7833.35417509, "command_buffers": 2 }, { "command_buffer_ns": 6458.31460133, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 10.691406714609375, 4.315755177234375, 4.11067685490625, 4.233072104390625, 4.0891927710625, 4.8059891923906255, 14.149739399734376, 4.093750249015625, 15.242839253921876 ], "command_buffer_throughput_us_p50": 4.315755177234375, "command_buffer_throughput_us_p90": 14.368359370571875, "command_buffer_latency_us": [ 6.624963134530001, 31.0833565891, 7.20837851986, 10.333373211300001, 7.75000080466, 6.500049494210001, 6.91669993103, 7.83335417509, 6.458314601330001 ], "command_buffer_latency_us_p50": 7.20837851986, "command_buffer_latency_us_p90": 14.483369886860004 }, "compute_throughput_us": [ 4.6640625, 4.61003125, 14.736984375, 4.80925, 15.319015625, 4.31315625, 4.80403125, 4.42578125, 4.768875 ], "compute_throughput_us_p50": 4.768875, "compute_throughput_us_p90": 14.853390625000001, "command_buffer_throughput_us": [ 4.666666427515626, 4.6113282223750005, 14.738281606703126, 4.809896381625, 15.321614228015626, 4.315754267734375, 4.8059891923906255, 4.427082785703125, 4.7688808990625 ], "command_buffer_throughput_us_p50": 4.7688808990625, "command_buffer_throughput_us_p90": 14.854948130965626, "compute_latency_us": [ 6.541, 31.25, 7.0, 6.5, 40.917, 7.334, 8.959, 228.041, 6.5 ], "compute_latency_us_p50": 7.334, "compute_latency_us_p90": 78.34180000000003, "command_buffer_latency_us": [ 6.54166797176, 31.3333584927, 7.1249669417699995, 6.54166797176, 40.9583444707, 7.41670373827, 9.08330548555, 228.041724768, 6.58328644931 ], "command_buffer_latency_us_p50": 7.41670373827, "command_buffer_latency_us_p90": 78.37502053016003, "counter_control_throughput_ratio": 1.0684869815368059, "counter_control_latency_ratio": 1.0193505358122787 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 6.3086094161, "throughput_us_p90": 6.319483219358, "latency_us_p50": 206.209, "latency_us_p90": 456.6836000000002 }, "torch": { "cold_call_ms": 4.5105, "warmup_ms": 200.965792, "repetitions": 2662, "throughput_us": [ 9.245210368144253, 9.221606686701728, 9.367502253944403, 9.375360255447031, 10.190927873779113, 10.248278362133734, 9.565755822689706, 9.344352742299023, 9.645473328324568 ], "latency_us": [ 347.792, 309.08299999999997, 234.08399999999997, 259.458, 289.625, 247.709, 235.083, 218.417, 229.25 ], "throughput_us_p50": 9.375360255447031, "throughput_us_p90": 10.202397971450038, "latency_us_p50": 247.709, "latency_us_p90": 316.8248, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 829541.6482724249, "command_buffers": 1 }, { "command_buffer_ns": 816083.3385773003, "command_buffers": 1 }, { "command_buffer_ns": 291041.68061167, "command_buffers": 1 }, { "command_buffer_ns": 305291.6727028787, "command_buffers": 1 }, { "command_buffer_ns": 304416.6951440275, "command_buffers": 1 }, { "command_buffer_ns": 1080750.022083521, "command_buffers": 1 }, { "command_buffer_ns": 290333.35158601403, "command_buffers": 1 }, { "command_buffer_ns": 728374.9873749912, "command_buffers": 1 }, { "command_buffer_ns": 288624.95673820376, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 234958.36649090052, "command_buffers": 1 }, { "command_buffer_ns": 8083.356078714132, "command_buffers": 1 }, { "command_buffer_ns": 23250.002413988113, "command_buffers": 1 }, { "command_buffer_ns": 8041.67939350009, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 7708.324119448662, "command_buffers": 1 }, { "command_buffer_ns": 8041.67939350009, "command_buffers": 1 }, { "command_buffer_ns": 9374.984074383974, "command_buffers": 1 }, { "command_buffer_ns": 7499.998901039362, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 12.96158825425664, 12.751302165270317, 4.547526259557344, 4.77018238598248, 4.756510861625429, 16.886719095055014, 4.536458618531469, 11.380859177734237, 4.509764949034434 ], "command_buffer_throughput_us_p50": 4.77018238598248, "command_buffer_throughput_us_p90": 13.746614422416314, "command_buffer_latency_us": [ 234.95836649090052, 8.083356078714132, 23.250002413988113, 8.04167939350009, 8.000002708286047, 7.708324119448662, 8.04167939350009, 9.374984074383974, 7.499998901039362 ], "command_buffer_latency_us_p50": 8.04167939350009, "command_buffer_latency_us_p90": 65.59167522937062 }, "throughput": [ { "compute_ns": 306916.0, "compute_span_ns": 306916.0, "command_buffer_ns": 306874.97928738594, "calibration_cpu_ns": 937417.0, "calibration_gpu_ticks": 937417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288959.0, "compute_span_ns": 288959.0, "command_buffer_ns": 288999.98869746923, "calibration_cpu_ns": 965209.0, "calibration_gpu_ticks": 965209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289000.0, "compute_span_ns": 289000.0, "command_buffer_ns": 288999.98869746923, "calibration_cpu_ns": 927625.0, "calibration_gpu_ticks": 927625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 719042.0, "compute_span_ns": 719042.0, "command_buffer_ns": 719083.3566710353, "calibration_cpu_ns": 2019125.0, "calibration_gpu_ticks": 2019125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 784833.0, "compute_span_ns": 784833.0, "command_buffer_ns": 784833.3334550261, "calibration_cpu_ns": 1944250.0, "calibration_gpu_ticks": 1944250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 290750.0, "compute_span_ns": 290750.0, "command_buffer_ns": 290750.00202283263, "calibration_cpu_ns": 2011541.0, "calibration_gpu_ticks": 2011541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 890834.0, "compute_span_ns": 890834.0, "command_buffer_ns": 890875.0023692846, "calibration_cpu_ns": 2857958.0, "calibration_gpu_ticks": 2857958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288542.0, "compute_span_ns": 288542.0, "command_buffer_ns": 288541.6615754366, "calibration_cpu_ns": 920917.0, "calibration_gpu_ticks": 920917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 297250.0, "compute_span_ns": 297250.0, "command_buffer_ns": 297249.9933093786, "calibration_cpu_ns": 1022834.0, "calibration_gpu_ticks": 1022834.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 9000.0, "compute_span_ns": 9000.0, "command_buffer_ns": 9000.010322779417, "calibration_cpu_ns": 632125.0, "calibration_gpu_ticks": 632125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7625.0, "compute_span_ns": 7625.0, "command_buffer_ns": 7625.02895668149, "calibration_cpu_ns": 427917.0, "calibration_gpu_ticks": 427917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9250.0, "compute_span_ns": 9250.0, "command_buffer_ns": 9250.01222640276, "calibration_cpu_ns": 793125.0, "calibration_gpu_ticks": 793125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7750.0, "compute_span_ns": 7750.0, "command_buffer_ns": 7750.0008046627045, "calibration_cpu_ns": 434125.0, "calibration_gpu_ticks": 434125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 248292.0, "compute_span_ns": 248292.0, "command_buffer_ns": 248291.70433804393, "calibration_cpu_ns": 1170334.0, "calibration_gpu_ticks": 1170334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9417.0, "compute_span_ns": 9417.0, "command_buffer_ns": 9416.660759598017, "calibration_cpu_ns": 645917.0, "calibration_gpu_ticks": 645917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8124.9745562672615, "calibration_cpu_ns": 458042.0, "calibration_gpu_ticks": 458042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 213375.0, "compute_span_ns": 213375.0, "command_buffer_ns": 213374.9658241868, "calibration_cpu_ns": 1303625.0, "calibration_gpu_ticks": 1303625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7750.0, "compute_span_ns": 7750.0, "command_buffer_ns": 7750.0008046627045, "calibration_cpu_ns": 445542.0, "calibration_gpu_ticks": 445542.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 4.7955625, 4.514984375, 4.515625, 11.23503125, 12.263015625, 4.54296875, 13.91928125, 4.50846875, 4.64453125 ], "compute_throughput_us_p50": 4.64453125, "compute_throughput_us_p90": 12.59426875, "command_buffer_throughput_us": [ 4.794921551365405, 4.515624823397957, 4.515624823397957, 11.235677447984926, 12.263020835234784, 4.54296878160676, 13.919921912020072, 4.508463462116197, 4.644531145459041 ], "command_buffer_throughput_us_p50": 4.644531145459041, "command_buffer_throughput_us_p90": 12.594401050591841, "compute_latency_us": [ 9.0, 7.625, 9.25, 7.75, 248.292, 9.417, 8.125, 213.375, 7.75 ], "compute_latency_us_p50": 9.0, "compute_latency_us_p90": 220.35840000000002, "command_buffer_latency_us": [ 9.000010322779417, 7.62502895668149, 9.25001222640276, 7.7500008046627045, 248.29170433804393, 9.416660759598017, 8.124974556267262, 213.3749658241868, 7.7500008046627045 ], "command_buffer_latency_us_p50": 9.000010322779417, "command_buffer_latency_us_p90": 220.35831352695823, "counter_control_throughput_ratio": 0.992984881375376, "counter_control_latency_ratio": 1.010357931309037 }, "download_ms": 0.319625, "allocation_upload_ms": 0.959833, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.6728924803113282, "device_compute_slowdown": 1.0267720773759463, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.246542, "warmup_ms": 200.140416, "repetitions": 1536, "throughput_us": [ 9.698215494791667, 9.211317057291668, 10.530219401041665, 10.545356119791666, 10.935981119791668, 10.906223307291667, 10.874321614583332, 10.836344401041666, 10.670247395833334 ], "latency_us": [ 308.334, 222.167, 300.666, 248.458, 260.291, 260.416, 219.125, 236.33399999999997, 223.208 ], "throughput_us_p50": 10.670247395833334, "throughput_us_p90": 10.912174869791667, "latency_us_p50": 248.458, "latency_us_p90": 302.1996, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 753458.2982771099, "command_buffers": 1 }, { "command_buffer_ns": 628750.0145845115, "command_buffers": 1 }, { "command_buffer_ns": 288916.69353470206, "command_buffers": 1 }, { "command_buffer_ns": 302625.00513345003, "command_buffers": 1 }, { "command_buffer_ns": 287750.03738701344, "command_buffers": 1 }, { "command_buffer_ns": 303833.3379663527, "command_buffers": 1 }, { "command_buffer_ns": 829000.0259876251, "command_buffers": 1 }, { "command_buffer_ns": 839791.6681133211, "command_buffers": 1 }, { "command_buffer_ns": 284666.66117310524, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 264083.3263285458, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 7750.0008046627045, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 9416.660759598017, "command_buffers": 1 }, { "command_buffer_ns": 7750.0008046627045, "command_buffers": 1 }, { "command_buffer_ns": 184874.9816417694, "command_buffers": 1 }, { "command_buffer_ns": 9124.982170760632, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 11.772785910579842, 9.824218977882992, 4.51432333647972, 4.728515705210157, 4.496094334172085, 4.747395905724261, 12.953125406056643, 13.121744814270642, 4.447916580829769 ], "command_buffer_throughput_us_p50": 4.747395905724261, "command_buffer_throughput_us_p90": 12.986849287699442, "command_buffer_latency_us": [ 264.0833263285458, 8.000002708286047, 8.124974556267262, 7.7500008046627045, 8.124974556267262, 9.416660759598017, 7.7500008046627045, 184.8749816417694, 9.124982170760632 ], "command_buffer_latency_us_p50": 8.124974556267262, "command_buffer_latency_us_p90": 200.7166505791247 }, "throughput": [ { "compute_ns": 289584.0, "compute_span_ns": 289584.0, "command_buffer_ns": 289624.96435269713, "calibration_cpu_ns": 936417.0, "calibration_gpu_ticks": 936417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288125.0, "compute_span_ns": 288125.0, "command_buffer_ns": 288125.011138618, "calibration_cpu_ns": 1124542.0, "calibration_gpu_ticks": 1124542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 297584.0, "compute_span_ns": 297584.0, "command_buffer_ns": 297625.0252686441, "calibration_cpu_ns": 1133584.0, "calibration_gpu_ticks": 1133584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 894042.0, "compute_span_ns": 894042.0, "command_buffer_ns": 894041.67374596, "calibration_cpu_ns": 1742625.0, "calibration_gpu_ticks": 1742625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 716667.0, "compute_span_ns": 716667.0, "command_buffer_ns": 716666.69100523, "calibration_cpu_ns": 2143333.0, "calibration_gpu_ticks": 2143333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288417.0, "compute_span_ns": 288417.0, "command_buffer_ns": 288458.3082050085, "calibration_cpu_ns": 978375.0, "calibration_gpu_ticks": 978375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289292.0, "compute_span_ns": 289292.0, "command_buffer_ns": 289291.6672863066, "calibration_cpu_ns": 953500.0, "calibration_gpu_ticks": 953500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289167.0, "compute_span_ns": 289167.0, "command_buffer_ns": 289166.6954383254, "calibration_cpu_ns": 880708.0, "calibration_gpu_ticks": 880708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288250.0, "compute_span_ns": 288250.0, "command_buffer_ns": 288249.9829865992, "calibration_cpu_ns": 872042.0, "calibration_gpu_ticks": 872042.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 9458.0, "compute_span_ns": 9458.0, "command_buffer_ns": 9416.660759598017, "calibration_cpu_ns": 627000.0, "calibration_gpu_ticks": 627000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7625.0, "compute_span_ns": 7625.0, "command_buffer_ns": 7624.9707490205765, "calibration_cpu_ns": 674291.0, "calibration_gpu_ticks": 674291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46042.0, "compute_span_ns": 46042.0, "command_buffer_ns": 46041.61949828267, "calibration_cpu_ns": 1356791.0, "calibration_gpu_ticks": 1356791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12916.0, "compute_span_ns": 12916.0, "command_buffer_ns": 12875.01072511077, "calibration_cpu_ns": 662000.0, "calibration_gpu_ticks": 662000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8209.0, "compute_span_ns": 8209.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 430541.0, "calibration_gpu_ticks": 430541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 219708.0, "compute_span_ns": 219708.0, "command_buffer_ns": 219708.30857753754, "calibration_cpu_ns": 1427292.0, "calibration_gpu_ticks": 1427292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7625.0, "compute_span_ns": 7625.0, "command_buffer_ns": 7625.02895668149, "calibration_cpu_ns": 444708.0, "calibration_gpu_ticks": 444708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9833.0, "compute_span_ns": 9833.0, "command_buffer_ns": 9791.692718863487, "calibration_cpu_ns": 724083.0, "calibration_gpu_ticks": 724083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7666.0, "compute_span_ns": 7666.0, "command_buffer_ns": 7624.9707490205765, "calibration_cpu_ns": 422250.0, "calibration_gpu_ticks": 422250.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 4.52475, 4.501953125, 4.64975, 13.96940625, 11.197921875, 4.506515625, 4.5201875, 4.518234375, 4.50390625 ], "compute_throughput_us_p50": 4.5201875, "compute_throughput_us_p90": 11.75221875, "command_buffer_throughput_us": [ 4.525390068010893, 4.501953299040906, 4.650391019822564, 13.969401152280625, 11.197917046956718, 4.507161065703258, 4.520182301348541, 4.5182296162238345, 4.503905984165613 ], "command_buffer_throughput_us_p50": 4.520182301348541, "command_buffer_throughput_us_p90": 11.7522138680215, "compute_latency_us": [ 9.458, 7.625, 46.042, 12.916, 8.209, 219.708, 7.625, 9.833, 7.666 ], "compute_latency_us_p50": 9.458, "compute_latency_us_p90": 80.77520000000004, "command_buffer_latency_us": [ 9.416660759598017, 7.6249707490205765, 46.04161949828267, 12.87501072511077, 8.25000461190939, 219.70830857753754, 7.62502895668149, 9.791692718863487, 7.6249707490205765 ], "command_buffer_latency_us_p50": 9.416660759598017, "command_buffer_latency_us_p90": 80.77495731413367, "counter_control_throughput_ratio": 0.949396501831386, "counter_control_latency_ratio": 0.9838746019347473 }, "download_ms": 0.335916, "allocation_upload_ms": 0.686292, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-s2w4dlh6/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "54d99166e42515c7bfff1cac173ad46750fa91f9c92e668c07d69318fdc050bf", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 24.571791, "capture_ms": 0.063375, "compile_ms": 56.550458, "allocation_upload_ms": 1.459625, "cold_call_ms": 2.034833, "warmup_ms": 200.218542, "download_ms": 0.436916, "repetitions": 4920, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 5.96831808943, 5.51560813008, 6.26676829268, 5.53616199187, 5.66664126016, 5.92042682927, 5.61729329268, 6.02071463415, 5.60112642276 ], "latency_us": [ 287.375, 251.625, 242.5, 207.791, 202.417, 195.625, 2024.333, 319.25, 321.292 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 56, "normalized_kernel_cost": 56, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 245209, "compute_span_ns": 245209, "calibration_cpu_ns": 782208, "calibration_gpu_ticks": 782208, "compute_passes": 1, "command_buffer_ns": 245291.623287, "command_buffers": 2 }, { "compute_ns": 243125, "compute_span_ns": 243125, "calibration_cpu_ns": 768459, "calibration_gpu_ticks": 768459, "compute_passes": 1, "command_buffer_ns": 243208.312895, "command_buffers": 2 }, { "compute_ns": 245209, "compute_span_ns": 245209, "calibration_cpu_ns": 705625, "calibration_gpu_ticks": 705625, "compute_passes": 1, "command_buffer_ns": 245375.034865, "command_buffers": 2 }, { "compute_ns": 252875, "compute_span_ns": 252875, "calibration_cpu_ns": 708708, "calibration_gpu_ticks": 708708, "compute_passes": 1, "command_buffer_ns": 253000.005614, "command_buffers": 2 }, { "compute_ns": 249750, "compute_span_ns": 249750, "calibration_cpu_ns": 793375, "calibration_gpu_ticks": 793375, "compute_passes": 1, "command_buffer_ns": 249833.27603, "command_buffers": 2 }, { "compute_ns": 593209, "compute_span_ns": 593209, "calibration_cpu_ns": 1859625, "calibration_gpu_ticks": 1859625, "compute_passes": 1, "command_buffer_ns": 593333.272263, "command_buffers": 2 }, { "compute_ns": 244125, "compute_span_ns": 244125, "calibration_cpu_ns": 666625, "calibration_gpu_ticks": 666625, "compute_passes": 1, "command_buffer_ns": 244250.055403, "command_buffers": 2 }, { "compute_ns": 882125, "compute_span_ns": 882125, "calibration_cpu_ns": 1731166, "calibration_gpu_ticks": 1731166, "compute_passes": 1, "command_buffer_ns": 882208.347321, "command_buffers": 2 }, { "compute_ns": 244875, "compute_span_ns": 244875, "calibration_cpu_ns": 694750, "calibration_gpu_ticks": 694750, "compute_passes": 1, "command_buffer_ns": 244874.97285, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7667, "compute_span_ns": 7667, "calibration_cpu_ns": 611416, "calibration_gpu_ticks": 611416, "compute_passes": 1, "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 399542, "calibration_gpu_ticks": 399542, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "compute_ns": 19208, "compute_span_ns": 19208, "calibration_cpu_ns": 853667, "calibration_gpu_ticks": 853667, "compute_passes": 1, "command_buffer_ns": 19333.3253264, "command_buffers": 2 }, { "compute_ns": 7459, "compute_span_ns": 7459, "calibration_cpu_ns": 503000, "calibration_gpu_ticks": 503000, "compute_passes": 1, "command_buffer_ns": 7583.29406381, "command_buffers": 2 }, { "compute_ns": 6750, "compute_span_ns": 6750, "calibration_cpu_ns": 400375, "calibration_gpu_ticks": 400375, "compute_passes": 1, "command_buffer_ns": 6791.61166772, "command_buffers": 2 }, { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 391541, "calibration_gpu_ticks": 391541, "compute_passes": 1, "command_buffer_ns": 6958.31840858, "command_buffers": 2 }, { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 474084, "calibration_gpu_ticks": 474084, "compute_passes": 1, "command_buffer_ns": 6791.66987538, "command_buffers": 2 }, { "compute_ns": 6334, "compute_span_ns": 6334, "calibration_cpu_ns": 389875, "calibration_gpu_ticks": 389875, "compute_passes": 1, "command_buffer_ns": 6416.69612378, "command_buffers": 2 }, { "compute_ns": 73625, "compute_span_ns": 73625, "calibration_cpu_ns": 1051250, "calibration_gpu_ticks": 1051250, "compute_passes": 1, "command_buffer_ns": 73749.9794923, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 848833.355121, "command_buffers": 2 }, { "command_buffer_ns": 243499.991484, "command_buffers": 2 }, { "command_buffer_ns": 840916.705783, "command_buffers": 2 }, { "command_buffer_ns": 821583.322249, "command_buffers": 2 }, { "command_buffer_ns": 283958.27394, "command_buffers": 2 }, { "command_buffer_ns": 265458.249487, "command_buffers": 2 }, { "command_buffer_ns": 269666.605163, "command_buffers": 2 }, { "command_buffer_ns": 262958.288658, "command_buffers": 2 }, { "command_buffer_ns": 261625.042185, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 228999.997489, "command_buffers": 2 }, { "command_buffer_ns": 6999.93688613, "command_buffers": 2 }, { "command_buffer_ns": 6875.02324581, "command_buffers": 2 }, { "command_buffer_ns": 6749.99319017, "command_buffers": 2 }, { "command_buffer_ns": 6458.37280899, "command_buffers": 2 }, { "command_buffer_ns": 69625.035394, "command_buffers": 2 }, { "command_buffer_ns": 7624.97074902, "command_buffers": 2 }, { "command_buffer_ns": 6833.3465606, "command_buffers": 2 }, { "command_buffer_ns": 7333.29216018, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 13.263021173765624, 3.8046873669375, 13.139323527859375, 12.837239410140624, 4.4368480303124995, 4.147785148234375, 4.213540705671875, 4.10872326028125, 4.087891284140625 ], "command_buffer_throughput_us_p50": 4.213540705671875, "command_buffer_throughput_us_p90": 13.164063057040625, "command_buffer_latency_us": [ 228.999997489, 6.9999368861299995, 6.87502324581, 6.7499931901700005, 6.458372808989999, 69.62503539400001, 7.62497074902, 6.8333465606, 7.33329216018 ], "command_buffer_latency_us_p50": 6.9999368861299995, "command_buffer_latency_us_p90": 101.50002781300003 }, "compute_throughput_us": [ 3.831390625, 3.798828125, 3.831390625, 3.951171875, 3.90234375, 9.268890625, 3.814453125, 13.783203125, 3.826171875 ], "compute_throughput_us_p50": 3.831390625, "compute_throughput_us_p90": 10.171753125, "command_buffer_throughput_us": [ 3.832681613859375, 3.800129888984375, 3.833984919765625, 3.95312508771875, 3.90364493796875, 9.270832379109375, 3.816407115671875, 13.784505426890624, 3.8261714507812496 ], "command_buffer_throughput_us_p50": 3.833984919765625, "command_buffer_throughput_us_p90": 10.173566988665625, "compute_latency_us": [ 7.667, 6.5, 19.208, 7.459, 6.75, 6.708, 6.708, 6.334, 73.625 ], "compute_latency_us_p50": 6.75, "compute_latency_us_p90": 30.091400000000007, "command_buffer_latency_us": [ 7.708324119449999, 6.500049494210001, 19.3333253264, 7.5832940638099995, 6.79161166772, 6.95831840858, 6.79166987538, 6.4166961237799995, 73.7499794923 ], "command_buffer_latency_us_p50": 6.95831840858, "command_buffer_latency_us_p90": 30.216656159580012, "counter_control_throughput_ratio": 0.9057482488621943, "counter_control_latency_ratio": 0.9390268833689277 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 5.66664126016, "throughput_us_p90": 6.069925365856, "latency_us_p50": 251.625, "latency_us_p90": 661.9002000000003 }, "slowdown": 0.5310693416886274, "device_compute_slowdown": 0.8476176320119465, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-5ijhkfzw/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b5a32df1046f38df4f8e9cbba5277df92f4beb4d4f9631ac4564ef5cda134264", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.461541, "capture_ms": 0.062125, "compile_ms": 55.984791, "allocation_upload_ms": 2.120833, "cold_call_ms": 4.214041, "warmup_ms": 200.831333, "download_ms": 0.924209, "repetitions": 1254, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 21.581539075, 20.8371212121, 20.8992886762, 20.8730398724, 22.981891547, 20.7150454545, 20.6769665072, 20.7233181818, 20.8193452951 ], "latency_us": [ 380.959, 541.667, 1640.25, 452.291, 306.167, 359.833, 240.917, 240.25, 208.292 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 68, "normalized_kernel_cost": 272, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 974583, "compute_span_ns": 974583, "calibration_cpu_ns": 1525500, "calibration_gpu_ticks": 1525500, "compute_passes": 1, "command_buffer_ns": 974708.353169, "command_buffers": 2 }, { "compute_ns": 947500, "compute_span_ns": 947500, "calibration_cpu_ns": 1425459, "calibration_gpu_ticks": 1425459, "compute_passes": 1, "command_buffer_ns": 947624.96883, "command_buffers": 2 }, { "compute_ns": 1497292, "compute_span_ns": 1497292, "calibration_cpu_ns": 2902834, "calibration_gpu_ticks": 2902834, "compute_passes": 1, "command_buffer_ns": 1497416.69884, "command_buffers": 2 }, { "compute_ns": 910375, "compute_span_ns": 910375, "calibration_cpu_ns": 1474333, "calibration_gpu_ticks": 1474333, "compute_passes": 1, "command_buffer_ns": 910416.652914, "command_buffers": 2 }, { "compute_ns": 1607333, "compute_span_ns": 1607333, "calibration_cpu_ns": 2576041, "calibration_gpu_ticks": 2576041, "compute_passes": 1, "command_buffer_ns": 1607416.66332, "command_buffers": 2 }, { "compute_ns": 905958, "compute_span_ns": 905958, "calibration_cpu_ns": 1378500, "calibration_gpu_ticks": 1378500, "compute_passes": 1, "command_buffer_ns": 905999.972019, "command_buffers": 2 }, { "compute_ns": 1938375, "compute_span_ns": 1938375, "calibration_cpu_ns": 3054333, "calibration_gpu_ticks": 3054333, "compute_passes": 1, "command_buffer_ns": 1938416.68079, "command_buffers": 2 }, { "compute_ns": 946584, "compute_span_ns": 946584, "calibration_cpu_ns": 1499167, "calibration_gpu_ticks": 1499167, "compute_passes": 1, "command_buffer_ns": 946750.049479, "command_buffers": 2 }, { "compute_ns": 1528000, "compute_span_ns": 1528000, "calibration_cpu_ns": 2396625, "calibration_gpu_ticks": 2396625, "compute_passes": 1, "command_buffer_ns": 1528083.34678, "command_buffers": 2 } ], "latency": [ { "compute_ns": 16333, "compute_span_ns": 16333, "calibration_cpu_ns": 473542, "calibration_gpu_ticks": 473542, "compute_passes": 1, "command_buffer_ns": 16291.6257977, "command_buffers": 2 }, { "compute_ns": 24500, "compute_span_ns": 24500, "calibration_cpu_ns": 1015917, "calibration_gpu_ticks": 1015917, "compute_passes": 1, "command_buffer_ns": 24625.0419877, "command_buffers": 2 }, { "compute_ns": 16208, "compute_span_ns": 16208, "calibration_cpu_ns": 387750, "calibration_gpu_ticks": 387750, "compute_passes": 1, "command_buffer_ns": 16166.6539498, "command_buffers": 2 }, { "compute_ns": 18709, "compute_span_ns": 18709, "calibration_cpu_ns": 760209, "calibration_gpu_ticks": 760209, "compute_passes": 1, "command_buffer_ns": 18833.3215192, "command_buffers": 2 }, { "compute_ns": 20792, "compute_span_ns": 20792, "calibration_cpu_ns": 669959, "calibration_gpu_ticks": 669959, "compute_passes": 1, "command_buffer_ns": 20874.9552257, "command_buffers": 2 }, { "compute_ns": 16416, "compute_span_ns": 16416, "calibration_cpu_ns": 438000, "calibration_gpu_ticks": 438000, "compute_passes": 1, "command_buffer_ns": 16416.7140611, "command_buffers": 2 }, { "compute_ns": 126875, "compute_span_ns": 126875, "calibration_cpu_ns": 1300041, "calibration_gpu_ticks": 1300041, "compute_passes": 1, "command_buffer_ns": 126916.682348, "command_buffers": 2 }, { "compute_ns": 21625, "compute_span_ns": 21625, "calibration_cpu_ns": 654375, "calibration_gpu_ticks": 654375, "compute_passes": 1, "command_buffer_ns": 21666.6376218, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 493666, "calibration_gpu_ticks": 493666, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1436125.02143, "command_buffers": 2 }, { "command_buffer_ns": 1679083.33242, "command_buffers": 2 }, { "command_buffer_ns": 913083.378691, "command_buffers": 2 }, { "command_buffer_ns": 1607416.60511, "command_buffers": 2 }, { "command_buffer_ns": 915999.989957, "command_buffers": 2 }, { "command_buffer_ns": 1515250.01274, "command_buffers": 2 }, { "command_buffer_ns": 1176333.34594, "command_buffers": 2 }, { "command_buffer_ns": 1836625.04423, "command_buffers": 2 }, { "command_buffer_ns": 919166.661333, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 16666.657757, "command_buffers": 2 }, { "command_buffer_ns": 48916.6122861, "command_buffers": 2 }, { "command_buffer_ns": 16374.9791682, "command_buffers": 2 }, { "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "command_buffer_ns": 214458.326809, "command_buffers": 2 }, { "command_buffer_ns": 18833.3215192, "command_buffers": 2 }, { "command_buffer_ns": 19041.6467376, "command_buffers": 2 }, { "command_buffer_ns": 16750.0693351, "command_buffers": 2 }, { "command_buffer_ns": 18541.6429304, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 22.43945345984375, 26.2356770690625, 14.266927792046875, 25.115884454843748, 14.312499843078125, 23.675781449062498, 18.380208530312498, 28.697266316093753, 14.361979083328126 ], "command_buffer_throughput_us_p50": 22.43945345984375, "command_buffer_throughput_us_p90": 26.727994918468752, "command_buffer_latency_us": [ 16.666657757, 48.9166122861, 16.3749791682, 18.541642930400002, 214.458326809, 18.8333215192, 19.0416467376, 16.7500693351, 18.541642930400002 ], "command_buffer_latency_us_p50": 18.541642930400002, "command_buffer_latency_us_p90": 82.02495519068003 }, "compute_throughput_us": [ 15.227859375, 14.8046875, 23.3951875, 14.224609375, 25.114578125, 14.15559375, 30.287109375, 14.790375, 23.875 ], "compute_throughput_us_p50": 15.227859375, "compute_throughput_us_p90": 26.149084375, "command_buffer_throughput_us": [ 15.229818018265625, 14.806640137968751, 23.397135919375, 14.22526020178125, 25.115885364375, 14.156249562796875, 30.287760637343748, 14.792969523109376, 23.8763022934375 ], "command_buffer_throughput_us_p50": 15.229818018265625, "command_buffer_throughput_us_p90": 26.15026041896875, "compute_latency_us": [ 16.333, 24.5, 16.208, 18.709, 20.792, 16.416, 126.875, 21.625, 18.5 ], "compute_latency_us_p50": 18.709, "compute_latency_us_p90": 44.97500000000002, "command_buffer_latency_us": [ 16.2916257977, 24.625041987699998, 16.1666539498, 18.8333215192, 20.8749552257, 16.4167140611, 126.91668234800001, 21.6666376218, 18.541642930400002 ], "command_buffer_latency_us_p50": 18.8333215192, "command_buffer_latency_us_p90": 45.083370059760014, "counter_control_throughput_ratio": 0.6787071728605137, "counter_control_latency_ratio": 0.9872778330732436 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 20.8371212121, "throughput_us_p90": 21.8616095694, "latency_us_p50": 359.833, "latency_us_p90": 761.3836000000002 }, "torch": { "cold_call_ms": 5.270042, "warmup_ms": 200.291292, "repetitions": 1142, "throughput_us": [ 27.546264448336252, 26.494964973730298, 26.979093695271455, 26.867703152364275, 27.284442206654994, 25.612558669001753, 27.05385288966725, 26.31819176882662, 25.902364273204903 ], "latency_us": [ 400.166, 290.45799999999997, 226.208, 220.542, 253.166, 287.66700000000003, 389.04100000000005, 278.708, 304.58299999999997 ], "throughput_us_p50": 26.867703152364275, "throughput_us_p90": 27.336806654991246, "latency_us_p50": 287.66700000000003, "latency_us_p90": 391.266, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1402375.0554770231, "command_buffers": 1 }, { "command_buffer_ns": 1036499.9761804938, "command_buffers": 1 }, { "command_buffer_ns": 1384083.298034966, "command_buffers": 1 }, { "command_buffer_ns": 1033750.013448298, "command_buffers": 1 }, { "command_buffer_ns": 1457499.9804608524, "command_buffers": 1 }, { "command_buffer_ns": 1035458.2918807864, "command_buffers": 1 }, { "command_buffer_ns": 1500916.6672825813, "command_buffers": 1 }, { "command_buffer_ns": 1044000.0332891941, "command_buffers": 1 }, { "command_buffer_ns": 1442791.661247611, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 252666.70854762197, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 20916.631910949945, "command_buffers": 1 }, { "command_buffer_ns": 18874.998204410076, "command_buffers": 1 }, { "command_buffer_ns": 21333.34055542946, "command_buffers": 1 }, { "command_buffer_ns": 23583.35768803954, "command_buffers": 1 }, { "command_buffer_ns": 18708.291463553905, "command_buffers": 1 }, { "command_buffer_ns": 30249.99750778079, "command_buffers": 1 }, { "command_buffer_ns": 180750.03754347563, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 21.912110241828486, 16.195312127820216, 21.626301531796344, 16.152343960129656, 22.77343719470082, 16.179035810637288, 23.451822926290333, 16.312500520143658, 22.543619706993923 ], "command_buffer_throughput_us_p50": 21.626301531796344, "command_buffer_throughput_us_p90": 22.90911434101872, "command_buffer_latency_us": [ 252.66670854762197, 20.99998528137803, 20.916631910949945, 18.874998204410076, 21.33334055542946, 23.58335768803954, 18.708291463553905, 30.24999750778079, 180.75003754347563 ], "command_buffer_latency_us_p50": 21.33334055542946, "command_buffer_latency_us_p90": 195.1333717443049 }, "throughput": [ { "compute_ns": 1034250.0, "compute_span_ns": 1034250.0, "command_buffer_ns": 1034250.0172555447, "calibration_cpu_ns": 1811791.0, "calibration_gpu_ticks": 1811791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1429625.0, "compute_span_ns": 1429625.0, "command_buffer_ns": 1429624.971933663, "calibration_cpu_ns": 2374000.0, "calibration_gpu_ticks": 2374000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1035250.0, "compute_span_ns": 1035250.0, "command_buffer_ns": 1035249.9666623771, "calibration_cpu_ns": 1751417.0, "calibration_gpu_ticks": 1751417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1441292.0, "compute_span_ns": 1441292.0, "command_buffer_ns": 1441333.326511085, "calibration_cpu_ns": 2800875.0, "calibration_gpu_ticks": 2800875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1054250.0, "compute_span_ns": 1054250.0, "command_buffer_ns": 1054249.9949224293, "calibration_cpu_ns": 1870667.0, "calibration_gpu_ticks": 1870667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1266791.0, "compute_span_ns": 1266791.0, "command_buffer_ns": 1266750.0413954258, "calibration_cpu_ns": 2172917.0, "calibration_gpu_ticks": 2172917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1041125.0, "compute_span_ns": 1041125.0, "command_buffer_ns": 1041124.9822936952, "calibration_cpu_ns": 1904250.0, "calibration_gpu_ticks": 1904250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1551958.0, "compute_span_ns": 1551958.0, "command_buffer_ns": 1551958.3830609918, "calibration_cpu_ns": 2470291.0, "calibration_gpu_ticks": 2470291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1033125.0, "compute_span_ns": 1033125.0, "command_buffer_ns": 1033124.9795854092, "calibration_cpu_ns": 1773500.0, "calibration_gpu_ticks": 1773500.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 24084.0, "compute_span_ns": 24084.0, "command_buffer_ns": 24124.979972839355, "calibration_cpu_ns": 751250.0, "calibration_gpu_ticks": 751250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 19042.0, "compute_span_ns": 19042.0, "command_buffer_ns": 19083.38163048029, "calibration_cpu_ns": 465708.0, "calibration_gpu_ticks": 465708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 204125.0, "compute_span_ns": 204125.0, "command_buffer_ns": 204125.01180544496, "calibration_cpu_ns": 1604000.0, "calibration_gpu_ticks": 1604000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 25541.0, "compute_span_ns": 25541.0, "command_buffer_ns": 25500.019546598196, "calibration_cpu_ns": 797000.0, "calibration_gpu_ticks": 797000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21000.0, "compute_span_ns": 21000.0, "command_buffer_ns": 20999.98528137803, "calibration_cpu_ns": 436750.0, "calibration_gpu_ticks": 436750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 100791.0, "compute_span_ns": 100791.0, "command_buffer_ns": 100750.01046061516, "calibration_cpu_ns": 1155042.0, "calibration_gpu_ticks": 1155042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20958.0, "compute_span_ns": 20958.0, "command_buffer_ns": 20958.308596163988, "calibration_cpu_ns": 426125.0, "calibration_gpu_ticks": 426125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20917.0, "compute_span_ns": 20917.0, "command_buffer_ns": 20916.69011861086, "calibration_cpu_ns": 573458.0, "calibration_gpu_ticks": 573458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24750.0, "compute_span_ns": 24750.0, "command_buffer_ns": 24750.01383572817, "calibration_cpu_ns": 753250.0, "calibration_gpu_ticks": 753250.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 16.16015625, 22.337890625, 16.17578125, 22.5201875, 16.47265625, 19.793609375, 16.267578125, 24.24934375, 16.142578125 ], "compute_throughput_us_p50": 16.47265625, "compute_throughput_us_p90": 22.86601875, "command_buffer_throughput_us": [ 16.160156519617885, 22.337890186463483, 16.175780729099642, 22.520833226735704, 16.472656170662958, 19.792969396803528, 16.267577848338988, 24.249349735327996, 16.142577806022018 ], "command_buffer_throughput_us_p50": 16.472656170662958, "command_buffer_throughput_us_p90": 22.866536528454162, "compute_latency_us": [ 24.084, 19.042, 204.125, 25.541, 21.0, 100.791, 20.958, 20.917, 24.75 ], "compute_latency_us_p50": 24.084, "compute_latency_us_p90": 121.45780000000002, "command_buffer_latency_us": [ 24.124979972839355, 19.08338163048029, 204.12501180544496, 25.500019546598196, 20.99998528137803, 100.75001046061516, 20.958308596163988, 20.91669011861086, 24.75001383572817 ], "command_buffer_latency_us_p50": 24.124979972839355, "command_buffer_latency_us_p90": 121.42501072958113, "counter_control_throughput_ratio": 0.7479679641623879, "counter_control_latency_ratio": 0.9843739768188069 }, "download_ms": 0.366834, "allocation_upload_ms": 1.552125, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7755453115562057, "device_compute_slowdown": 0.9244325349774721, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.388375, "warmup_ms": 200.086791, "repetitions": 964, "throughput_us": [ 26.59664522821577, 26.827196058091285, 26.00553215767635, 25.775414937759336, 25.909060165975106, 25.958290456431538, 25.838692946058092, 25.84936929460581, 25.615534232365146 ], "latency_us": [ 458.25, 269.5, 2129.25, 386.084, 309.417, 336.95799999999997, 324.20799999999997, 247.25, 232.958 ], "throughput_us_p50": 25.909060165975106, "throughput_us_p90": 26.642755394190875, "latency_us_p50": 324.20799999999997, "latency_us_p90": 792.4500000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1070458.3255574107, "command_buffers": 1 }, { "command_buffer_ns": 1625875.011086464, "command_buffers": 1 }, { "command_buffer_ns": 1032375.0320822, "command_buffers": 1 }, { "command_buffer_ns": 1427374.9548010528, "command_buffers": 1 }, { "command_buffer_ns": 1041333.3657197654, "command_buffers": 1 }, { "command_buffer_ns": 1531374.9900087714, "command_buffers": 1 }, { "command_buffer_ns": 1057874.9934211373, "command_buffers": 1 }, { "command_buffer_ns": 1578083.320055157, "command_buffers": 1 }, { "command_buffer_ns": 1035833.3820477128, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 18750.02635642886, "command_buffers": 1 }, { "command_buffer_ns": 22791.67529195547, "command_buffers": 1 }, { "command_buffer_ns": 18874.998204410076, "command_buffers": 1 }, { "command_buffer_ns": 273666.6356213391, "command_buffers": 1 }, { "command_buffer_ns": 24500.011932104826, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 140333.3735652268, "command_buffers": 1 }, { "command_buffer_ns": 21249.987185001373, "command_buffers": 1 }, { "command_buffer_ns": 21041.661966592073, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.725911336834542, 25.404297048226, 16.130859876284376, 22.30273366876645, 16.270833839371335, 23.927734218887053, 16.52929677220527, 24.657551875861827, 16.184896594495513 ], "command_buffer_throughput_us_p50": 16.725911336834542, "command_buffer_throughput_us_p90": 24.80690091033466, "command_buffer_latency_us": [ 18.75002635642886, 22.79167529195547, 18.874998204410076, 273.6666356213391, 24.500011932104826, 20.99998528137803, 140.3333735652268, 21.249987185001373, 21.041661966592073 ], "command_buffer_latency_us_p50": 21.249987185001373, "command_buffer_latency_us_p90": 167.00002597644928 }, "throughput": [ { "compute_ns": 1585250.0, "compute_span_ns": 1585250.0, "command_buffer_ns": 1585250.0218898058, "calibration_cpu_ns": 2910375.0, "calibration_gpu_ticks": 2910375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1036709.0, "compute_span_ns": 1036709.0, "command_buffer_ns": 1036750.0362917781, "calibration_cpu_ns": 1724875.0, "calibration_gpu_ticks": 1724875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1545042.0, "compute_span_ns": 1545042.0, "command_buffer_ns": 1545041.6831299663, "calibration_cpu_ns": 2658583.0, "calibration_gpu_ticks": 2658583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1050292.0, "compute_span_ns": 1050292.0, "command_buffer_ns": 1050291.64114967, "calibration_cpu_ns": 1815375.0, "calibration_gpu_ticks": 1815375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1434542.0, "compute_span_ns": 1434542.0, "command_buffer_ns": 1434541.6566357017, "calibration_cpu_ns": 2727666.0, "calibration_gpu_ticks": 2727666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1030334.0, "compute_span_ns": 1030334.0, "command_buffer_ns": 1030374.9586455524, "calibration_cpu_ns": 1685333.0, "calibration_gpu_ticks": 1685333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1491084.0, "compute_span_ns": 1491084.0, "command_buffer_ns": 1491124.9745637178, "calibration_cpu_ns": 2686667.0, "calibration_gpu_ticks": 2686667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1140458.0, "compute_span_ns": 1140458.0, "command_buffer_ns": 1140416.6580177844, "calibration_cpu_ns": 1747500.0, "calibration_gpu_ticks": 1747500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1496583.0, "compute_span_ns": 1496583.0, "command_buffer_ns": 1496583.3397582173, "calibration_cpu_ns": 2677417.0, "calibration_gpu_ticks": 2677417.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 18959.0, "compute_span_ns": 18959.0, "command_buffer_ns": 19000.028260052204, "calibration_cpu_ns": 467542.0, "calibration_gpu_ticks": 467542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 214167.0, "compute_span_ns": 214167.0, "command_buffer_ns": 214208.32490548491, "calibration_cpu_ns": 1233750.0, "calibration_gpu_ticks": 1233750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18791.0, "compute_span_ns": 18791.0, "command_buffer_ns": 18750.02635642886, "calibration_cpu_ns": 545208.0, "calibration_gpu_ticks": 545208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21083.0, "compute_span_ns": 21083.0, "command_buffer_ns": 21041.661966592073, "calibration_cpu_ns": 530208.0, "calibration_gpu_ticks": 530208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18833.0, "compute_span_ns": 18833.0, "command_buffer_ns": 18833.321519196033, "calibration_cpu_ns": 502125.0, "calibration_gpu_ticks": 502125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20917.0, "compute_span_ns": 20917.0, "command_buffer_ns": 20916.69011861086, "calibration_cpu_ns": 542458.0, "calibration_gpu_ticks": 542458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22625.0, "compute_span_ns": 22625.0, "command_buffer_ns": 22624.9685510993, "calibration_cpu_ns": 740000.0, "calibration_gpu_ticks": 740000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 18584.0, "compute_span_ns": 18584.0, "command_buffer_ns": 18624.996300786734, "calibration_cpu_ns": 464833.0, "calibration_gpu_ticks": 464833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125667.0, "compute_span_ns": 125667.0, "command_buffer_ns": 125708.34951475263, "calibration_cpu_ns": 1387917.0, "calibration_gpu_ticks": 1387917.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 24.76953125, 16.198578125, 24.14128125, 16.4108125, 22.41471875, 16.09896875, 23.2981875, 17.81965625, 23.384109375 ], "compute_throughput_us_p50": 22.41471875, "compute_throughput_us_p90": 24.26693125, "command_buffer_throughput_us": [ 24.769531592028216, 16.199219317059033, 24.141276298905723, 16.410806892963592, 22.41471338493284, 16.099608728836756, 23.29882772755809, 17.81901028152788, 23.384114683722146 ], "command_buffer_throughput_us_p50": 22.41471338493284, "command_buffer_throughput_us_p90": 24.26692735753022, "compute_latency_us": [ 18.959, 214.167, 18.791, 21.083, 18.833, 20.917, 22.625, 18.584, 125.667 ], "compute_latency_us_p50": 20.917, "compute_latency_us_p90": 143.36700000000002, "command_buffer_latency_us": [ 19.000028260052204, 214.20832490548491, 18.75002635642886, 21.041661966592073, 18.833321519196033, 20.91669011861086, 22.6249685510993, 18.624996300786734, 125.70834951475263 ], "command_buffer_latency_us_p50": 20.91669011861086, "command_buffer_latency_us_p90": 143.40834459289908, "counter_control_throughput_ratio": 1.377600779788855, "counter_control_latency_ratio": 0.9933789743115305 }, "download_ms": 0.503667, "allocation_upload_ms": 0.689917, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-d_of3u67/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "a8c235e7c8be57d97298e1f9f70525f6c2c7f4ae4cee9647b15503a46aab5ade", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.521625, "capture_ms": 0.05975, "compile_ms": 60.368, "allocation_upload_ms": 3.7405, "cold_call_ms": 5.411958, "warmup_ms": 202.722375, "download_ms": 0.787166, "repetitions": 1202, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 21.2658760399, 21.5927279534, 21.4755266223, 22.6314126456, 21.0144201331, 21.354687188, 21.1523843594, 21.5057545757, 21.4260257903 ], "latency_us": [ 325.916, 256.542, 243, 269.458, 250.833, 303.5, 313.625, 262.333, 296.459 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 60, "normalized_kernel_cost": 240, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 974750, "compute_span_ns": 974750, "calibration_cpu_ns": 1476166, "calibration_gpu_ticks": 1476166, "compute_passes": 1, "command_buffer_ns": 974875.001702, "command_buffers": 2 }, { "compute_ns": 1567791, "compute_span_ns": 1567791, "calibration_cpu_ns": 2710584, "calibration_gpu_ticks": 2710584, "compute_passes": 1, "command_buffer_ns": 1567875.03511, "command_buffers": 2 }, { "compute_ns": 1007000, "compute_span_ns": 1007000, "calibration_cpu_ns": 2445958, "calibration_gpu_ticks": 2445958, "compute_passes": 1, "command_buffer_ns": 1007041.66107, "command_buffers": 2 }, { "compute_ns": 1949208, "compute_span_ns": 1949208, "calibration_cpu_ns": 2772333, "calibration_gpu_ticks": 2772333, "compute_passes": 1, "command_buffer_ns": 1949208.38112, "command_buffers": 2 }, { "compute_ns": 958583, "compute_span_ns": 958583, "calibration_cpu_ns": 1490750, "calibration_gpu_ticks": 1490750, "compute_passes": 1, "command_buffer_ns": 958624.994382, "command_buffers": 2 }, { "compute_ns": 1517500, "compute_span_ns": 1517500, "calibration_cpu_ns": 2349750, "calibration_gpu_ticks": 2349750, "compute_passes": 1, "command_buffer_ns": 1517625.05993, "command_buffers": 2 }, { "compute_ns": 970125, "compute_span_ns": 970125, "calibration_cpu_ns": 1663917, "calibration_gpu_ticks": 1663917, "compute_passes": 1, "command_buffer_ns": 970124.965534, "command_buffers": 2 }, { "compute_ns": 1591208, "compute_span_ns": 1591208, "calibration_cpu_ns": 2430958, "calibration_gpu_ticks": 2430958, "compute_passes": 1, "command_buffer_ns": 1591250.00937, "command_buffers": 2 }, { "compute_ns": 993917, "compute_span_ns": 993917, "calibration_cpu_ns": 1681292, "calibration_gpu_ticks": 1681292, "compute_passes": 1, "command_buffer_ns": 994041.678496, "command_buffers": 2 } ], "latency": [ { "compute_ns": 24541, "compute_span_ns": 24541, "calibration_cpu_ns": 426667, "calibration_gpu_ticks": 426667, "compute_passes": 1, "command_buffer_ns": 24541.6886173, "command_buffers": 2 }, { "compute_ns": 17750, "compute_span_ns": 17750, "calibration_cpu_ns": 421958, "calibration_gpu_ticks": 421958, "compute_passes": 1, "command_buffer_ns": 17874.9905899, "command_buffers": 2 }, { "compute_ns": 193750, "compute_span_ns": 193750, "calibration_cpu_ns": 1549125, "calibration_gpu_ticks": 1549125, "compute_passes": 1, "command_buffer_ns": 193875.050172, "command_buffers": 2 }, { "compute_ns": 21541, "compute_span_ns": 21541, "calibration_cpu_ns": 662375, "calibration_gpu_ticks": 662375, "compute_passes": 1, "command_buffer_ns": 21583.2842514, "command_buffers": 2 }, { "compute_ns": 60333, "compute_span_ns": 60333, "calibration_cpu_ns": 1305125, "calibration_gpu_ticks": 1305125, "compute_passes": 1, "command_buffer_ns": 60333.3464824, "command_buffers": 2 }, { "compute_ns": 22666, "compute_span_ns": 22666, "calibration_cpu_ns": 618750, "calibration_gpu_ticks": 618750, "compute_passes": 1, "command_buffer_ns": 22708.3219215, "command_buffers": 2 }, { "compute_ns": 19167, "compute_span_ns": 19167, "calibration_cpu_ns": 471500, "calibration_gpu_ticks": 471500, "compute_passes": 1, "command_buffer_ns": 19208.2952708, "command_buffers": 2 }, { "compute_ns": 267459, "compute_span_ns": 267459, "calibration_cpu_ns": 1046500, "calibration_gpu_ticks": 1046500, "compute_passes": 1, "command_buffer_ns": 267499.999609, "command_buffers": 2 }, { "compute_ns": 19125, "compute_span_ns": 19125, "calibration_cpu_ns": 582500, "calibration_gpu_ticks": 582500, "compute_passes": 1, "command_buffer_ns": 19208.3534785, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1476500.00872, "command_buffers": 2 }, { "command_buffer_ns": 949499.954004, "command_buffers": 2 }, { "command_buffer_ns": 1667124.97594, "command_buffers": 2 }, { "command_buffer_ns": 952125.061303, "command_buffers": 2 }, { "command_buffer_ns": 1711833.34896, "command_buffers": 2 }, { "command_buffer_ns": 978041.673079, "command_buffers": 2 }, { "command_buffer_ns": 1452458.32391, "command_buffers": 2 }, { "command_buffer_ns": 936708.354857, "command_buffers": 2 }, { "command_buffer_ns": 1614999.95738, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 60458.3183303, "command_buffers": 2 }, { "command_buffer_ns": 17374.9867827, "command_buffers": 2 }, { "command_buffer_ns": 16625.0392795, "command_buffers": 2 }, { "command_buffer_ns": 17166.6615643, "command_buffers": 2 }, { "command_buffer_ns": 19083.3234228, "command_buffers": 2 }, { "command_buffer_ns": 17333.3100975, "command_buffers": 2 }, { "command_buffer_ns": 19125.000108, "command_buffers": 2 }, { "command_buffer_ns": 22708.3219215, "command_buffers": 2 }, { "command_buffer_ns": 16458.3325386, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 23.07031263625, 14.835936781312501, 26.0488277490625, 14.876954082859374, 26.747396077500003, 15.281901141859375, 22.69466131109375, 14.636068044640625, 25.2343743340625 ], "command_buffer_throughput_us_p50": 22.69466131109375, "command_buffer_throughput_us_p90": 26.188541414750002, "command_buffer_latency_us": [ 60.458318330299996, 17.374986782700002, 16.6250392795, 17.1666615643, 19.0833234228, 17.3333100975, 19.125000108000002, 22.708321921499998, 16.4583325386 ], "command_buffer_latency_us_p50": 17.374986782700002, "command_buffer_latency_us_p90": 30.258321203260003 }, "compute_throughput_us": [ 15.23046875, 24.496734375, 15.734375, 30.456375, 14.977859375, 23.7109375, 15.158203125, 24.862625, 15.529953125 ], "compute_throughput_us_p50": 15.734375, "compute_throughput_us_p90": 25.981375000000003, "command_buffer_throughput_us": [ 15.232421901593751, 24.49804742359375, 15.73502595421875, 30.456380955, 14.97851553721875, 23.71289156140625, 15.15820258646875, 24.86328139640625, 15.5319012265 ], "command_buffer_throughput_us_p50": 15.73502595421875, "command_buffer_throughput_us_p90": 25.981901308125003, "compute_latency_us": [ 24.541, 17.75, 193.75, 21.541, 60.333, 22.666, 19.167, 267.459, 19.125 ], "compute_latency_us_p50": 22.666, "compute_latency_us_p90": 208.4918, "command_buffer_latency_us": [ 24.5416886173, 17.8749905899, 193.875050172, 21.583284251400002, 60.333346482399996, 22.708321921499998, 19.2082952708, 267.499999609, 19.2083534785 ], "command_buffer_latency_us_p50": 22.708321921499998, "command_buffer_latency_us_p90": 208.6000400594, "counter_control_throughput_ratio": 0.6679193127706655, "counter_control_latency_ratio": 1.2572790679513868 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 21.4260257903, "throughput_us_p90": 21.80046489184, "latency_us_p50": 269.458, "latency_us_p90": 316.0832 }, "slowdown": 0.8269703977312762, "device_compute_slowdown": 0.701966202453466, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-ozwuei4e/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "c9fd11968a5edbf77e894438f42e4bd465ad146629716a6f08eacac9ab82063f", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.857875, "capture_ms": 0.068208, "compile_ms": 57.183875, "allocation_upload_ms": 7.983708, "cold_call_ms": 8.608084, "warmup_ms": 200.784875, "download_ms": 3.601875, "repetitions": 317, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 92.4931640379, 98.6436624606, 90.4108832808, 98.9944794953, 98.5059148265, 90.7632776025, 99.3600157729, 99.5176119874, 90.7359369085 ], "latency_us": [ 2047.709, 656.375, 442.708, 253.5, 279.208, 263.375, 265.667, 368.417, 311.542 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 96, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4666458, "compute_span_ns": 4666458, "calibration_cpu_ns": 5326125, "calibration_gpu_ticks": 5326125, "compute_passes": 1, "command_buffer_ns": 4666500.02636, "command_buffers": 2 }, { "compute_ns": 4729667, "compute_span_ns": 4729667, "calibration_cpu_ns": 5870834, "calibration_gpu_ticks": 5870834, "compute_passes": 1, "command_buffer_ns": 4729791.66079, "command_buffers": 2 }, { "compute_ns": 4938000, "compute_span_ns": 4938000, "calibration_cpu_ns": 5913250, "calibration_gpu_ticks": 5913250, "compute_passes": 1, "command_buffer_ns": 4938041.6749, "command_buffers": 2 }, { "compute_ns": 4942458, "compute_span_ns": 4942458, "calibration_cpu_ns": 6002167, "calibration_gpu_ticks": 6002167, "compute_passes": 1, "command_buffer_ns": 4942458.29759, "command_buffers": 2 }, { "compute_ns": 4762083, "compute_span_ns": 4762083, "calibration_cpu_ns": 5610792, "calibration_gpu_ticks": 5610792, "compute_passes": 1, "command_buffer_ns": 4762124.96869, "command_buffers": 2 }, { "compute_ns": 4837000, "compute_span_ns": 4837000, "calibration_cpu_ns": 5737709, "calibration_gpu_ticks": 5737709, "compute_passes": 1, "command_buffer_ns": 4837083.28102, "command_buffers": 2 }, { "compute_ns": 4827292, "compute_span_ns": 4827292, "calibration_cpu_ns": 5827333, "calibration_gpu_ticks": 5827333, "compute_passes": 1, "command_buffer_ns": 4827333.32319, "command_buffers": 2 }, { "compute_ns": 6929292, "compute_span_ns": 6929292, "calibration_cpu_ns": 8340584, "calibration_gpu_ticks": 8340584, "compute_passes": 1, "command_buffer_ns": 6929333.32175, "command_buffers": 2 }, { "compute_ns": 6643167, "compute_span_ns": 6643167, "calibration_cpu_ns": 7673708, "calibration_gpu_ticks": 7673708, "compute_passes": 1, "command_buffer_ns": 6643208.32584, "command_buffers": 2 } ], "latency": [ { "compute_ns": 65417, "compute_span_ns": 65417, "calibration_cpu_ns": 487709, "calibration_gpu_ticks": 487709, "compute_passes": 1, "command_buffer_ns": 65416.6797176, "command_buffers": 2 }, { "compute_ns": 115833, "compute_span_ns": 115833, "calibration_cpu_ns": 2866459, "calibration_gpu_ticks": 2866459, "compute_passes": 1, "command_buffer_ns": 115833.361633, "command_buffers": 2 }, { "compute_ns": 117167, "compute_span_ns": 117167, "calibration_cpu_ns": 2167375, "calibration_gpu_ticks": 2167375, "compute_passes": 1, "command_buffer_ns": 117291.69637, "command_buffers": 2 }, { "compute_ns": 66875, "compute_span_ns": 66875, "calibration_cpu_ns": 473958, "calibration_gpu_ticks": 473958, "compute_passes": 1, "command_buffer_ns": 66916.6329317, "command_buffers": 2 }, { "compute_ns": 518917, "compute_span_ns": 518917, "calibration_cpu_ns": 1407125, "calibration_gpu_ticks": 1407125, "compute_passes": 1, "command_buffer_ns": 519041.670486, "command_buffers": 2 }, { "compute_ns": 115625, "compute_span_ns": 115625, "calibration_cpu_ns": 768917, "calibration_gpu_ticks": 768917, "compute_passes": 1, "command_buffer_ns": 115749.950055, "command_buffers": 2 }, { "compute_ns": 738458, "compute_span_ns": 738458, "calibration_cpu_ns": 2027875, "calibration_gpu_ticks": 2027875, "compute_passes": 1, "command_buffer_ns": 738541.653845, "command_buffers": 2 }, { "compute_ns": 99625, "compute_span_ns": 99625, "calibration_cpu_ns": 561791, "calibration_gpu_ticks": 561791, "compute_passes": 1, "command_buffer_ns": 99708.2679532, "command_buffers": 2 }, { "compute_ns": 317250, "compute_span_ns": 317250, "calibration_cpu_ns": 1477875, "calibration_gpu_ticks": 1477875, "compute_passes": 1, "command_buffer_ns": 317333.324347, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4585916.70031, "command_buffers": 2 }, { "command_buffer_ns": 4830333.28783, "command_buffers": 2 }, { "command_buffer_ns": 4987458.291, "command_buffers": 2 }, { "command_buffer_ns": 4938999.94763, "command_buffers": 2 }, { "command_buffer_ns": 4800874.97272, "command_buffers": 2 }, { "command_buffer_ns": 4839750.00679, "command_buffers": 2 }, { "command_buffer_ns": 4947083.2455, "command_buffers": 2 }, { "command_buffer_ns": 4142875.02645, "command_buffers": 2 }, { "command_buffer_ns": 4574208.28752, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 99916.6513793, "command_buffers": 2 }, { "command_buffer_ns": 65958.3020024, "command_buffers": 2 }, { "command_buffer_ns": 66374.9524392, "command_buffers": 2 }, { "command_buffer_ns": 66958.3096169, "command_buffers": 2 }, { "command_buffer_ns": 69083.2966939, "command_buffers": 2 }, { "command_buffer_ns": 73458.3009034, "command_buffers": 2 }, { "command_buffer_ns": 106374.965981, "command_buffers": 2 }, { "command_buffer_ns": 66666.6310281, "command_buffers": 2 }, { "command_buffer_ns": 66083.332058, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 71.65494844234375, 75.47395762234375, 77.929035796875, 77.17187418171875, 75.01367144875, 75.62109385609375, 77.2981757109375, 64.73242228828124, 71.47200449249999 ], "command_buffer_throughput_us_p50": 75.47395762234375, "command_buffer_throughput_us_p90": 77.424347728125, "command_buffer_latency_us": [ 99.9166513793, 65.9583020024, 66.3749524392, 66.95830961690001, 69.08329669390001, 73.4583009034, 106.374965981, 66.6666310281, 66.083332058 ], "command_buffer_latency_us_p50": 66.95830961690001, "command_buffer_latency_us_p90": 101.20831429964001 }, "compute_throughput_us": [ 72.91340625, 73.901046875, 77.15625, 77.22590625, 74.407546875, 75.578125, 75.4264375, 108.2701875, 103.799484375 ], "compute_throughput_us_p50": 75.578125, "compute_throughput_us_p90": 104.69362500000001, "command_buffer_throughput_us": [ 72.91406291187499, 73.90299469984375, 77.1569011703125, 77.22591089984374, 74.40820263578124, 75.5794262659375, 75.42708317484374, 108.27083315234376, 103.80013009125 ], "command_buffer_throughput_us_p50": 75.5794262659375, "command_buffer_throughput_us_p90": 104.69427070346875, "compute_latency_us": [ 65.417, 115.833, 117.167, 66.875, 518.917, 115.625, 738.458, 99.625, 317.25 ], "compute_latency_us_p50": 115.833, "compute_latency_us_p90": 562.8252000000001, "command_buffer_latency_us": [ 65.41667971759999, 115.833361633, 117.29169637000001, 66.91663293170001, 519.041670486, 115.749950055, 738.5416538449999, 99.70826795320001, 317.333324347 ], "command_buffer_latency_us_p50": 115.833361633, "command_buffer_latency_us_p90": 562.9416671578001, "counter_control_throughput_ratio": 0.9994489951410178, "counter_control_latency_ratio": 1.7561604546579324 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 98.5059148265, "throughput_us_p90": 99.3915350158, "latency_us_p50": 311.542, "latency_us_p90": 934.6418000000003 }, "torch": { "cold_call_ms": 9.126416, "warmup_ms": 200.493958, "repetitions": 255, "throughput_us": [ 116.43415294117646, 120.90963921568627, 112.02075294117648, 123.40506666666667, 115.08398823529411, 117.98071764705884, 122.98610980392156, 112.08022745098039, 123.74101568627451 ], "latency_us": [ 474.58299999999997, 382.167, 349.916, 337.875, 371.458, 430.75, 2027.584, 972.0, 408.58299999999997 ], "throughput_us_p50": 117.98071764705884, "throughput_us_p90": 123.47225647058823, "latency_us_p50": 408.58299999999997, "latency_us_p90": 1183.1168000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7519749.982748181, "command_buffers": 1 }, { "command_buffer_ns": 7603125.006426126, "command_buffers": 1 }, { "command_buffer_ns": 5579083.284828812, "command_buffers": 1 }, { "command_buffer_ns": 5119458.364788443, "command_buffers": 1 }, { "command_buffer_ns": 5227249.988820404, "command_buffers": 1 }, { "command_buffer_ns": 5291374.982334673, "command_buffers": 1 }, { "command_buffer_ns": 5328833.300154656, "command_buffers": 1 }, { "command_buffer_ns": 5295291.71762988, "command_buffers": 1 }, { "command_buffer_ns": 5205833.294894546, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 524375.0056251884, "command_buffers": 1 }, { "command_buffer_ns": 83999.99933317304, "command_buffers": 1 }, { "command_buffer_ns": 113833.34640413523, "command_buffers": 1 }, { "command_buffer_ns": 127583.33468809724, "command_buffers": 1 }, { "command_buffer_ns": 517125.00862777233, "command_buffers": 1 }, { "command_buffer_ns": 80541.64936766028, "command_buffers": 1 }, { "command_buffer_ns": 519666.6461415589, "command_buffers": 1 }, { "command_buffer_ns": 80124.99893084168, "command_buffers": 1 }, { "command_buffer_ns": 584624.9987371266, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 117.49609348044032, 118.79882822540822, 87.17317632545019, 79.99153694981942, 81.67578107531881, 82.67773409897927, 83.2630203149165, 82.73893308796687, 81.34114523272729 ], "command_buffer_throughput_us_p50": 82.73893308796687, "command_buffer_throughput_us_p90": 117.7566404294339, "command_buffer_latency_us": [ 524.3750056251884, 83.99999933317304, 113.83334640413523, 127.58333468809724, 517.1250086277723, 80.54164936766028, 519.6666461415589, 80.12499893084168, 584.6249987371266 ], "command_buffer_latency_us_p50": 127.58333468809724, "command_buffer_latency_us_p90": 536.425004247576 }, "throughput": [ { "compute_ns": 7537584.0, "compute_span_ns": 7537584.0, "command_buffer_ns": 7537624.973338097, "calibration_cpu_ns": 8359917.0, "calibration_gpu_ticks": 8359917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 7575500.0, "compute_span_ns": 7575500.0, "command_buffer_ns": 7575499.99980256, "calibration_cpu_ns": 8500458.0, "calibration_gpu_ticks": 8500458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5255375.0, "compute_span_ns": 5255375.0, "command_buffer_ns": 5255374.999251217, "calibration_cpu_ns": 6602417.0, "calibration_gpu_ticks": 6602417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5339875.0, "compute_span_ns": 5339875.0, "command_buffer_ns": 5339874.944183975, "calibration_cpu_ns": 6776708.0, "calibration_gpu_ticks": 6776708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5550083.0, "compute_span_ns": 5550083.0, "command_buffer_ns": 5550041.678361595, "calibration_cpu_ns": 6862541.0, "calibration_gpu_ticks": 6862541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5545791.0, "compute_span_ns": 5545791.0, "command_buffer_ns": 5545749.969314784, "calibration_cpu_ns": 7021333.0, "calibration_gpu_ticks": 7021333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5254792.0, "compute_span_ns": 5254792.0, "command_buffer_ns": 5254791.700281203, "calibration_cpu_ns": 6641792.0, "calibration_gpu_ticks": 6641792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5535083.0, "compute_span_ns": 5535083.0, "command_buffer_ns": 5535083.35724473, "calibration_cpu_ns": 7048500.0, "calibration_gpu_ticks": 7048500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5476667.0, "compute_span_ns": 5476667.0, "command_buffer_ns": 5476666.672620922, "calibration_cpu_ns": 6608000.0, "calibration_gpu_ticks": 6608000.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 107416.0, "compute_span_ns": 107416.0, "command_buffer_ns": 107375.03180280328, "calibration_cpu_ns": 585916.0, "calibration_gpu_ticks": 585916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81958.0, "compute_span_ns": 81958.0, "command_buffer_ns": 81916.63073375821, "calibration_cpu_ns": 696875.0, "calibration_gpu_ticks": 696875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80375.0, "compute_span_ns": 80375.0, "command_buffer_ns": 80375.00083446503, "calibration_cpu_ns": 608792.0, "calibration_gpu_ticks": 608792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80542.0, "compute_span_ns": 80542.0, "command_buffer_ns": 80541.64936766028, "calibration_cpu_ns": 529292.0, "calibration_gpu_ticks": 529292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 112625.0, "compute_span_ns": 112625.0, "command_buffer_ns": 112625.01357123256, "calibration_cpu_ns": 541083.0, "calibration_gpu_ticks": 541083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 96500.0, "compute_span_ns": 96500.0, "command_buffer_ns": 96500.03630667925, "calibration_cpu_ns": 657792.0, "calibration_gpu_ticks": 657792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 114791.0, "compute_span_ns": 114791.0, "command_buffer_ns": 114750.00064820051, "calibration_cpu_ns": 602000.0, "calibration_gpu_ticks": 602000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80541.0, "compute_span_ns": 80541.0, "command_buffer_ns": 80500.03089010715, "calibration_cpu_ns": 601250.0, "calibration_gpu_ticks": 601250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 114625.0, "compute_span_ns": 114625.0, "command_buffer_ns": 114625.0288002193, "calibration_cpu_ns": 644791.0, "calibration_gpu_ticks": 644791.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 117.77475, 118.3671875, 82.115234375, 83.435546875, 86.720046875, 86.652984375, 82.106125, 86.485671875, 85.572921875 ], "compute_throughput_us_p50": 86.485671875, "compute_throughput_us_p90": 117.8932375, "command_buffer_throughput_us": [ 117.77539020840777, 118.367187496915, 82.11523436330026, 83.43554600287462, 86.71940122439992, 86.6523432705435, 82.1061203168938, 86.4856774569489, 85.57291675970191 ], "command_buffer_throughput_us_p50": 86.4856774569489, "command_buffer_throughput_us_p90": 117.89374966610922, "compute_latency_us": [ 107.416, 81.958, 80.375, 80.542, 112.625, 96.5, 114.791, 80.541, 114.625 ], "compute_latency_us_p50": 96.5, "compute_latency_us_p90": 114.6582, "command_buffer_latency_us": [ 107.37503180280328, 81.91663073375821, 80.37500083446503, 80.54164936766028, 112.62501357123256, 96.50003630667925, 114.75000064820051, 80.50003089010715, 114.6250288002193 ], "command_buffer_latency_us_p50": 96.50003630667925, "command_buffer_latency_us_p90": 114.65002316981554, "counter_control_throughput_ratio": 1.04305466783587, "counter_control_latency_ratio": 0.6312865983990802 }, "download_ms": 0.951708, "allocation_upload_ms": 19.448041, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.8349323244598493, "device_compute_slowdown": 0.8738803013432681, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.502792, "warmup_ms": 200.274458, "repetitions": 265, "throughput_us": [ 120.74858490566038, 120.44653962264151, 111.2228, 118.98412075471698, 120.51509433962264, 120.45487547169812, 110.74402264150943, 120.46981132075472, 120.37625660377358 ], "latency_us": [ 432.79200000000003, 372.875, 356.04200000000003, 460.0, 344.041, 353.95799999999997, 333.41700000000003, 2358.833, 1053.0 ], "throughput_us_p50": 120.44653962264151, "throughput_us_p90": 120.56179245283019, "latency_us_p50": 372.875, "latency_us_p90": 1314.1666000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 6302791.7058207095, "command_buffers": 1 }, { "command_buffer_ns": 5706791.649572551, "command_buffers": 1 }, { "command_buffer_ns": 5326458.311174065, "command_buffers": 1 }, { "command_buffer_ns": 5464625.02097711, "command_buffers": 1 }, { "command_buffer_ns": 5627541.686408222, "command_buffers": 1 }, { "command_buffer_ns": 5723083.333577961, "command_buffers": 1 }, { "command_buffer_ns": 5283000.005874783, "command_buffers": 1 }, { "command_buffer_ns": 5510208.31335336, "command_buffers": 1 }, { "command_buffer_ns": 5533249.9905489385, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 385250.02310052514, "command_buffers": 1 }, { "command_buffer_ns": 80583.32605287433, "command_buffers": 1 }, { "command_buffer_ns": 576749.9678768218, "command_buffers": 1 }, { "command_buffer_ns": 80208.29409360886, "command_buffers": 1 }, { "command_buffer_ns": 510250.0435896218, "command_buffers": 1 }, { "command_buffer_ns": 82541.66459664702, "command_buffers": 1 }, { "command_buffer_ns": 80250.02898648381, "command_buffers": 1 }, { "command_buffer_ns": 80333.32414925098, "command_buffers": 1 }, { "command_buffer_ns": 79916.61550477147, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 98.48112040344859, 89.16861952457111, 83.22591111209476, 85.38476595276734, 87.93033885012846, 89.42317708715564, 82.54687509179348, 86.09700489614625, 86.45703110232716 ], "command_buffer_throughput_us_p50": 86.45703110232716, "command_buffer_throughput_us_p90": 91.23476575041423, "command_buffer_latency_us": [ 385.25002310052514, 80.58332605287433, 576.7499678768218, 80.20829409360886, 510.2500435896218, 82.54166459664702, 80.25002898648381, 80.33332414925098, 79.91661550477147 ], "command_buffer_latency_us_p50": 80.58332605287433, "command_buffer_latency_us_p90": 523.5500284470618 }, "throughput": [ { "compute_ns": 5129958.0, "compute_span_ns": 5129958.0, "command_buffer_ns": 5129916.651640087, "calibration_cpu_ns": 6531959.0, "calibration_gpu_ticks": 6531959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5213167.0, "compute_span_ns": 5213167.0, "command_buffer_ns": 5213208.321947604, "calibration_cpu_ns": 6623167.0, "calibration_gpu_ticks": 6623167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5281000.0, "compute_span_ns": 5281000.0, "command_buffer_ns": 5280999.990645796, "calibration_cpu_ns": 6503125.0, "calibration_gpu_ticks": 6503125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5422917.0, "compute_span_ns": 5422917.0, "command_buffer_ns": 5422958.289273083, "calibration_cpu_ns": 6509833.0, "calibration_gpu_ticks": 6509833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5435459.0, "compute_span_ns": 5435459.0, "command_buffer_ns": 5435500.002931803, "calibration_cpu_ns": 7034250.0, "calibration_gpu_ticks": 7034250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5153250.0, "compute_span_ns": 5153250.0, "command_buffer_ns": 5153250.007424504, "calibration_cpu_ns": 6669500.0, "calibration_gpu_ticks": 6669500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5198750.0, "compute_span_ns": 5198750.0, "command_buffer_ns": 5198750.004637986, "calibration_cpu_ns": 6699250.0, "calibration_gpu_ticks": 6699250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5358375.0, "compute_span_ns": 5358375.0, "command_buffer_ns": 5358374.968636781, "calibration_cpu_ns": 6545292.0, "calibration_gpu_ticks": 6545292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5638250.0, "compute_span_ns": 5638250.0, "command_buffer_ns": 5638250.0333711505, "calibration_cpu_ns": 6600250.0, "calibration_gpu_ticks": 6600250.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 110125.0, "compute_span_ns": 110125.0, "command_buffer_ns": 110124.99453499913, "calibration_cpu_ns": 632417.0, "calibration_gpu_ticks": 632417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81708.0, "compute_span_ns": 81708.0, "command_buffer_ns": 81708.30551534891, "calibration_cpu_ns": 654833.0, "calibration_gpu_ticks": 654833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 112167.0, "compute_span_ns": 112167.0, "command_buffer_ns": 112166.68644919991, "calibration_cpu_ns": 568541.0, "calibration_gpu_ticks": 568541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 83250.0, "compute_span_ns": 83250.0, "command_buffer_ns": 83249.99362230301, "calibration_cpu_ns": 483542.0, "calibration_gpu_ticks": 483542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 108833.0, "compute_span_ns": 108833.0, "command_buffer_ns": 108833.30833166838, "calibration_cpu_ns": 573125.0, "calibration_gpu_ticks": 573125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80167.0, "compute_span_ns": 80167.0, "command_buffer_ns": 80166.67561605573, "calibration_cpu_ns": 496709.0, "calibration_gpu_ticks": 496709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 516334.0, "compute_span_ns": 516334.0, "command_buffer_ns": 516375.0029169023, "calibration_cpu_ns": 1771666.0, "calibration_gpu_ticks": 1771666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 112334.0, "compute_span_ns": 112334.0, "command_buffer_ns": 112375.01166760921, "calibration_cpu_ns": 556459.0, "calibration_gpu_ticks": 556459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 577709.0, "compute_span_ns": 577709.0, "command_buffer_ns": 577749.9754913151, "calibration_cpu_ns": 1563958.0, "calibration_gpu_ticks": 1563958.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 80.15559375, 81.455734375, 82.515625, 84.733078125, 84.929046875, 80.51953125, 81.23046875, 83.724609375, 88.09765625 ], "compute_throughput_us_p50": 82.515625, "compute_throughput_us_p90": 85.56276875, "command_buffer_throughput_us": [ 80.15494768187637, 81.45638003043132, 82.51562485384056, 84.73372326989193, 84.92968754580943, 80.51953136600787, 81.23046882246854, 83.7246088849497, 88.09765677142423 ], "command_buffer_throughput_us_p50": 82.51562485384056, "command_buffer_throughput_us_p90": 85.56328139093239, "compute_latency_us": [ 110.125, 81.708, 112.167, 83.25, 108.833, 80.167, 516.334, 112.334, 577.709 ], "compute_latency_us_p50": 110.125, "compute_latency_us_p90": 528.6089999999999, "command_buffer_latency_us": [ 110.12499453499913, 81.70830551534891, 112.16668644919991, 83.24999362230301, 108.83330833166838, 80.16667561605573, 516.3750029169023, 112.37501166760921, 577.7499754913151 ], "command_buffer_latency_us_p50": 110.12499453499913, "command_buffer_latency_us_p90": 528.6499974317849, "counter_control_throughput_ratio": 0.9724450808241445, "counter_control_latency_ratio": 1.013960449606367 }, "download_ms": 0.777667, "allocation_upload_ms": 1.549875, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-6vvbrr7s/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "232522da57db77c64b377ec2facdc2e1df33cd57064a4297b89d6582fd4ad63b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.855291, "capture_ms": 0.064375, "compile_ms": 56.00125, "allocation_upload_ms": 6.3065, "cold_call_ms": 5.226417, "warmup_ms": 200.461542, "download_ms": 3.359208, "repetitions": 341, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 84.0399560117, 95.175829912, 91.6786422287, 88.2385131965, 86.7085777126, 92.059627566, 83.7453577713, 92.4188651026, 91.5876099707 ], "latency_us": [ 387.875, 319.334, 283.833, 267.708, 343.875, 338.458, 297.25, 276.208, 278.125 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 104, "normalized_kernel_cost": 1248, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4289667, "compute_span_ns": 4289667, "calibration_cpu_ns": 5350250, "calibration_gpu_ticks": 5350250, "compute_passes": 1, "command_buffer_ns": 4289833.36315, "command_buffers": 2 }, { "compute_ns": 4626209, "compute_span_ns": 4626209, "calibration_cpu_ns": 5447084, "calibration_gpu_ticks": 5447084, "compute_passes": 1, "command_buffer_ns": 4626333.30608, "command_buffers": 2 }, { "compute_ns": 4564250, "compute_span_ns": 4564250, "calibration_cpu_ns": 5466250, "calibration_gpu_ticks": 5466250, "compute_passes": 1, "command_buffer_ns": 4564374.97633, "command_buffers": 2 }, { "compute_ns": 4732375, "compute_span_ns": 4732375, "calibration_cpu_ns": 5615041, "calibration_gpu_ticks": 5615041, "compute_passes": 1, "command_buffer_ns": 4732500.00505, "command_buffers": 2 }, { "compute_ns": 4409542, "compute_span_ns": 4409542, "calibration_cpu_ns": 5450875, "calibration_gpu_ticks": 5450875, "compute_passes": 1, "command_buffer_ns": 4409583.34366, "command_buffers": 2 }, { "compute_ns": 4518875, "compute_span_ns": 4518875, "calibration_cpu_ns": 5538875, "calibration_gpu_ticks": 5538875, "compute_passes": 1, "command_buffer_ns": 4518916.6558, "command_buffers": 2 }, { "compute_ns": 4454000, "compute_span_ns": 4454000, "calibration_cpu_ns": 5505792, "calibration_gpu_ticks": 5505792, "compute_passes": 1, "command_buffer_ns": 4454125.00994, "command_buffers": 2 }, { "compute_ns": 4477291, "compute_span_ns": 4477291, "calibration_cpu_ns": 5450625, "calibration_gpu_ticks": 5450625, "compute_passes": 1, "command_buffer_ns": 4477291.65899, "command_buffers": 2 }, { "compute_ns": 4337708, "compute_span_ns": 4337708, "calibration_cpu_ns": 5397791, "calibration_gpu_ticks": 5397791, "compute_passes": 1, "command_buffer_ns": 4337791.6445, "command_buffers": 2 } ], "latency": [ { "compute_ns": 108792, "compute_span_ns": 108792, "calibration_cpu_ns": 567750, "calibration_gpu_ticks": 567750, "compute_passes": 1, "command_buffer_ns": 108916.661702, "command_buffers": 2 }, { "compute_ns": 61833, "compute_span_ns": 61833, "calibration_cpu_ns": 551667, "calibration_gpu_ticks": 551667, "compute_passes": 1, "command_buffer_ns": 61875.0345893, "command_buffers": 2 }, { "compute_ns": 106834, "compute_span_ns": 106834, "calibration_cpu_ns": 697583, "calibration_gpu_ticks": 697583, "compute_passes": 1, "command_buffer_ns": 106916.646473, "command_buffers": 2 }, { "compute_ns": 61500, "compute_span_ns": 61500, "calibration_cpu_ns": 545250, "calibration_gpu_ticks": 545250, "compute_passes": 1, "command_buffer_ns": 61583.3560005, "command_buffers": 2 }, { "compute_ns": 112667, "compute_span_ns": 112667, "calibration_cpu_ns": 649417, "calibration_gpu_ticks": 649417, "compute_passes": 1, "command_buffer_ns": 112791.662104, "command_buffers": 2 }, { "compute_ns": 61208, "compute_span_ns": 61208, "calibration_cpu_ns": 501459, "calibration_gpu_ticks": 501459, "compute_passes": 1, "command_buffer_ns": 61291.6774116, "command_buffers": 2 }, { "compute_ns": 552834, "compute_span_ns": 552834, "calibration_cpu_ns": 2004459, "calibration_gpu_ticks": 2004459, "compute_passes": 1, "command_buffer_ns": 552999.961656, "command_buffers": 2 }, { "compute_ns": 107791, "compute_span_ns": 107791, "calibration_cpu_ns": 600583, "calibration_gpu_ticks": 600583, "compute_passes": 1, "command_buffer_ns": 107750.005554, "command_buffers": 2 }, { "compute_ns": 508208, "compute_span_ns": 508208, "calibration_cpu_ns": 2031208, "calibration_gpu_ticks": 2031208, "compute_passes": 1, "command_buffer_ns": 508249.970153, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4436958.34838, "command_buffers": 2 }, { "command_buffer_ns": 4508000.04182, "command_buffers": 2 }, { "command_buffer_ns": 4624958.32471, "command_buffers": 2 }, { "command_buffer_ns": 4550958.34332, "command_buffers": 2 }, { "command_buffer_ns": 4529166.61743, "command_buffers": 2 }, { "command_buffer_ns": 4459125.04802, "command_buffers": 2 }, { "command_buffer_ns": 4410749.99981, "command_buffers": 2 }, { "command_buffer_ns": 4654750.05331, "command_buffers": 2 }, { "command_buffer_ns": 4597916.67527, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 562375.003938, "command_buffers": 2 }, { "command_buffer_ns": 61500.0026301, "command_buffers": 2 }, { "command_buffer_ns": 475041.684695, "command_buffers": 2 }, { "command_buffer_ns": 70124.9809936, "command_buffers": 2 }, { "command_buffer_ns": 493416.679092, "command_buffers": 2 }, { "command_buffer_ns": 61083.2939856, "command_buffers": 2 }, { "command_buffer_ns": 61166.647356, "command_buffers": 2 }, { "command_buffer_ns": 61499.9444224, "command_buffers": 2 }, { "command_buffer_ns": 61083.3521932, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 69.32747419343751, 70.4375006534375, 72.26497382359375, 71.108724114375, 70.76822839734375, 69.6738288753125, 68.91796874703125, 72.73046958296875, 71.84244805109374 ], "command_buffer_throughput_us_p50": 70.76822839734375, "command_buffer_throughput_us_p90": 72.35807297546874, "command_buffer_latency_us": [ 562.375003938, 61.500002630100006, 475.041684695, 70.12498099359999, 493.416679092, 61.0832939856, 61.166647356, 61.499944422400006, 61.0833521932 ], "command_buffer_latency_us_p50": 61.500002630100006, "command_buffer_latency_us_p90": 507.20834406119997 }, "compute_throughput_us": [ 67.026046875, 72.284515625, 71.31640625, 73.943359375, 68.89909375, 70.607421875, 69.59375, 69.957671875, 67.7766875 ], "compute_throughput_us_p50": 69.957671875, "compute_throughput_us_p90": 72.61628437499999, "command_buffer_throughput_us": [ 67.02864629921875, 72.2864579075, 71.31835900515625, 73.94531257890625, 68.8997397446875, 70.608072746875, 69.5957032803125, 69.95768217171876, 67.7779944453125 ], "command_buffer_throughput_us_p50": 69.95768217171876, "command_buffer_throughput_us_p90": 72.61822884178125, "compute_latency_us": [ 108.792, 61.833, 106.834, 61.5, 112.667, 61.208, 552.834, 107.791, 508.208 ], "compute_latency_us_p50": 107.791, "compute_latency_us_p90": 517.1332, "command_buffer_latency_us": [ 108.916661702, 61.8750345893, 106.916646473, 61.583356000500004, 112.791662104, 61.2916774116, 552.999961656, 107.750005554, 508.249970153 ], "command_buffer_latency_us_p50": 107.750005554, "command_buffer_latency_us_p90": 517.1999684536, "counter_control_throughput_ratio": 0.9869007796121495, "counter_control_latency_ratio": 1.0034114634690317 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 91.5876099707, "throughput_us_p90": 92.97025806447999, "latency_us_p50": 297.25, "latency_us_p90": 352.675 }, "slowdown": 0.7604005084549842, "device_compute_slowdown": 0.8478112099981064, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-7hux9_87/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "26111b6b600e52c50494ad3f783ef6fc3b02252d329d47ea39eb66749773d18e", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.894542, "capture_ms": 0.067166, "compile_ms": 55.425125, "allocation_upload_ms": 6.435166, "cold_call_ms": 6.343625, "warmup_ms": 200.021333, "download_ms": 2.405167, "repetitions": 398, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 72.5146582915, 78.9267160804, 79.3195150754, 87.2384849246, 79.3714396985, 73.5617688442, 76.4571809045, 66.7130452261, 74.4564497487 ], "latency_us": [ 539.75, 427.834, 451.917, 393.917, 388.916, 331.958, 320, 295.958, 314.125 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 1280, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3821958, "compute_span_ns": 3821958, "calibration_cpu_ns": 4774084, "calibration_gpu_ticks": 4774084, "compute_passes": 1, "command_buffer_ns": 3822041.67545, "command_buffers": 2 }, { "compute_ns": 4130208, "compute_span_ns": 4130208, "calibration_cpu_ns": 4995083, "calibration_gpu_ticks": 4995083, "compute_passes": 1, "command_buffer_ns": 4130166.66425, "command_buffers": 2 }, { "compute_ns": 3996083, "compute_span_ns": 3996083, "calibration_cpu_ns": 4925334, "calibration_gpu_ticks": 4925334, "compute_passes": 1, "command_buffer_ns": 3996083.28007, "command_buffers": 2 }, { "compute_ns": 4062375, "compute_span_ns": 4062375, "calibration_cpu_ns": 4973958, "calibration_gpu_ticks": 4973958, "compute_passes": 1, "command_buffer_ns": 4062374.99556, "command_buffers": 2 }, { "compute_ns": 4049834, "compute_span_ns": 4049834, "calibration_cpu_ns": 5098667, "calibration_gpu_ticks": 5098667, "compute_passes": 1, "command_buffer_ns": 4049916.63527, "command_buffers": 2 }, { "compute_ns": 3926750, "compute_span_ns": 3926750, "calibration_cpu_ns": 5135833, "calibration_gpu_ticks": 5135833, "compute_passes": 1, "command_buffer_ns": 3926749.98147, "command_buffers": 2 }, { "compute_ns": 4095875, "compute_span_ns": 4095875, "calibration_cpu_ns": 4927584, "calibration_gpu_ticks": 4927584, "compute_passes": 1, "command_buffer_ns": 4095875.01781, "command_buffers": 2 }, { "compute_ns": 3879708, "compute_span_ns": 3879708, "calibration_cpu_ns": 5217000, "calibration_gpu_ticks": 5217000, "compute_passes": 1, "command_buffer_ns": 3879791.59132, "command_buffers": 2 }, { "compute_ns": 3348917, "compute_span_ns": 3348917, "calibration_cpu_ns": 4188417, "calibration_gpu_ticks": 4188417, "compute_passes": 1, "command_buffer_ns": 3349041.62446, "command_buffers": 2 } ], "latency": [ { "compute_ns": 59750, "compute_span_ns": 59750, "calibration_cpu_ns": 562041, "calibration_gpu_ticks": 562041, "compute_passes": 1, "command_buffer_ns": 59749.9893047, "command_buffers": 2 }, { "compute_ns": 63542, "compute_span_ns": 63542, "calibration_cpu_ns": 512291, "calibration_gpu_ticks": 512291, "compute_passes": 1, "command_buffer_ns": 63624.9314994, "command_buffers": 2 }, { "compute_ns": 102916, "compute_span_ns": 102916, "calibration_cpu_ns": 607708, "calibration_gpu_ticks": 607708, "compute_passes": 1, "command_buffer_ns": 102916.674223, "command_buffers": 2 }, { "compute_ns": 60959, "compute_span_ns": 60959, "calibration_cpu_ns": 542708, "calibration_gpu_ticks": 542708, "compute_passes": 1, "command_buffer_ns": 61083.3521932, "command_buffers": 2 }, { "compute_ns": 54750, "compute_span_ns": 54750, "calibration_cpu_ns": 508792, "calibration_gpu_ticks": 508792, "compute_passes": 1, "command_buffer_ns": 54833.3046027, "command_buffers": 2 }, { "compute_ns": 413625, "compute_span_ns": 413625, "calibration_cpu_ns": 1548667, "calibration_gpu_ticks": 1548667, "compute_passes": 1, "command_buffer_ns": 413708.388805, "command_buffers": 2 }, { "compute_ns": 60542, "compute_span_ns": 60542, "calibration_cpu_ns": 741583, "calibration_gpu_ticks": 741583, "compute_passes": 1, "command_buffer_ns": 60583.2901783, "command_buffers": 2 }, { "compute_ns": 411750, "compute_span_ns": 411750, "calibration_cpu_ns": 1168750, "calibration_gpu_ticks": 1168750, "compute_passes": 1, "command_buffer_ns": 411875.02211, "command_buffers": 2 }, { "compute_ns": 53834, "compute_span_ns": 53834, "calibration_cpu_ns": 481708, "calibration_gpu_ticks": 481708, "compute_passes": 1, "command_buffer_ns": 54000.003729, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3435375.0525, "command_buffers": 2 }, { "command_buffer_ns": 4008999.96748, "command_buffers": 2 }, { "command_buffer_ns": 4012499.99413, "command_buffers": 2 }, { "command_buffer_ns": 4025208.35632, "command_buffers": 2 }, { "command_buffer_ns": 4070583.38169, "command_buffers": 2 }, { "command_buffer_ns": 4059916.6532, "command_buffers": 2 }, { "command_buffer_ns": 4090500.00599, "command_buffers": 2 }, { "command_buffer_ns": 6155958.28276, "command_buffers": 2 }, { "command_buffer_ns": 3283083.32246, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 98208.3729468, "command_buffers": 2 }, { "command_buffer_ns": 101749.959867, "command_buffers": 2 }, { "command_buffer_ns": 69000.0015311, "command_buffers": 2 }, { "command_buffer_ns": 99249.9408312, "command_buffers": 2 }, { "command_buffer_ns": 66458.3058096, "command_buffers": 2 }, { "command_buffer_ns": 95875.0024438, "command_buffers": 2 }, { "command_buffer_ns": 53874.9736734, "command_buffers": 2 }, { "command_buffer_ns": 99541.6776277, "command_buffers": 2 }, { "command_buffer_ns": 54000.003729, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 53.6777351953125, 62.640624491875, 62.695312408281254, 62.8938805675, 63.602865338906255, 63.43619770625, 63.91406259359375, 96.186848168125, 51.2981769134375 ], "command_buffer_throughput_us_p50": 62.8938805675, "command_buffer_throughput_us_p90": 70.3686197085, "command_buffer_latency_us": [ 98.2083729468, 101.749959867, 69.0000015311, 99.2499408312, 66.4583058096, 95.8750024438, 53.87497367340001, 99.5416776277, 54.000003729 ], "command_buffer_latency_us_p50": 95.8750024438, "command_buffer_latency_us_p90": 99.98333407556 }, "compute_throughput_us": [ 59.71809375, 64.5345, 62.438796875, 63.474609375, 63.27865625, 61.35546875, 63.998046875, 60.6204375, 52.326828125 ], "compute_throughput_us_p50": 62.438796875, "compute_throughput_us_p90": 64.1053375, "command_buffer_throughput_us": [ 59.71940117890625, 64.53385412890624, 62.43880125109375, 63.474609305625, 63.27994742609375, 61.35546846046875, 63.99804715328125, 60.621743614375, 52.3287753821875 ], "command_buffer_throughput_us_p50": 62.43880125109375, "command_buffer_throughput_us_p90": 64.10520854840625, "compute_latency_us": [ 59.75, 63.542, 102.916, 60.959, 54.75, 413.625, 60.542, 411.75, 53.834 ], "compute_latency_us_p50": 60.959, "compute_latency_us_p90": 412.125, "command_buffer_latency_us": [ 59.7499893047, 63.6249314994, 102.916674223, 61.0833521932, 54.833304602700004, 413.70838880499997, 60.5832901783, 411.87502211000003, 54.000003729 ], "command_buffer_latency_us_p50": 61.0833521932, "command_buffer_latency_us_p90": 412.241695449, "counter_control_throughput_ratio": 1.0013140231786162, "counter_control_latency_ratio": 1.0 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 76.4571809045, "throughput_us_p90": 80.94484874372, "latency_us_p50": 388.916, "latency_us_p90": 469.4836 }, "torch": { "cold_call_ms": 7.849125, "warmup_ms": 200.733625, "repetitions": 257, "throughput_us": [ 109.15840077821012, 102.36624513618678, 101.59192607003891, 101.90061867704281, 103.14834630350195, 106.2057392996109, 101.77836964980546, 102.19049805447472, 102.15515564202335 ], "latency_us": [ 436.0, 314.083, 344.75, 306.458, 310.125, 303.333, 313.583, 2170.292, 1071.4170000000001 ], "throughput_us_p50": 102.19049805447472, "throughput_us_p90": 106.79627159533074, "latency_us_p50": 314.083, "latency_us_p90": 1291.1920000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4935791.715979576, "command_buffers": 1 }, { "command_buffer_ns": 4690999.980084598, "command_buffers": 1 }, { "command_buffer_ns": 5176291.684620082, "command_buffers": 1 }, { "command_buffer_ns": 4916166.653856635, "command_buffers": 1 }, { "command_buffer_ns": 4735500.027891248, "command_buffers": 1 }, { "command_buffer_ns": 4833458.340726793, "command_buffers": 1 }, { "command_buffer_ns": 4747500.002849847, "command_buffers": 1 }, { "command_buffer_ns": 4882749.984972179, "command_buffers": 1 }, { "command_buffer_ns": 4754166.700877249, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 421916.65852442384, "command_buffers": 1 }, { "command_buffer_ns": 71416.66719689965, "command_buffers": 1 }, { "command_buffer_ns": 416583.3233855665, "command_buffers": 1 }, { "command_buffer_ns": 70750.01485645771, "command_buffers": 1 }, { "command_buffer_ns": 376874.9884329736, "command_buffers": 1 }, { "command_buffer_ns": 71541.69725254178, "command_buffers": 1 }, { "command_buffer_ns": 71416.66719689965, "command_buffers": 1 }, { "command_buffer_ns": 71124.98860806227, "command_buffers": 1 }, { "command_buffer_ns": 70750.01485645771, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 77.12174556218088, 73.29687468882184, 80.87955757218879, 76.81510396650992, 73.99218793580076, 75.52278657385614, 74.17968754452886, 76.2929685151903, 74.28385470120702 ], "command_buffer_throughput_us_p50": 75.52278657385614, "command_buffer_throughput_us_p90": 77.87330796418246, "command_buffer_latency_us": [ 421.91665852442384, 71.41666719689965, 416.5833233855665, 70.75001485645771, 376.8749884329736, 71.54169725254178, 71.41666719689965, 71.12498860806227, 70.75001485645771 ], "command_buffer_latency_us_p50": 71.41666719689965, "command_buffer_latency_us_p90": 417.64999041333795 }, "throughput": [ { "compute_ns": 4654084.0, "compute_span_ns": 4654084.0, "command_buffer_ns": 4654124.961234629, "calibration_cpu_ns": 5555250.0, "calibration_gpu_ticks": 5555250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4835458.0, "compute_span_ns": 4835458.0, "command_buffer_ns": 4835458.297748119, "calibration_cpu_ns": 5731042.0, "calibration_gpu_ticks": 5731042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4455291.0, "compute_span_ns": 4455291.0, "command_buffer_ns": 4455249.989405274, "calibration_cpu_ns": 6264666.0, "calibration_gpu_ticks": 6264666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4688292.0, "compute_span_ns": 4688292.0, "command_buffer_ns": 4688333.312515169, "calibration_cpu_ns": 6102000.0, "calibration_gpu_ticks": 6102000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4518167.0, "compute_span_ns": 4518167.0, "command_buffer_ns": 4518208.326771855, "calibration_cpu_ns": 5845500.0, "calibration_gpu_ticks": 5845500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4736291.0, "compute_span_ns": 4736291.0, "command_buffer_ns": 4736250.0336021185, "calibration_cpu_ns": 5930875.0, "calibration_gpu_ticks": 5930875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4748625.0, "compute_span_ns": 4748625.0, "command_buffer_ns": 4748625.040519983, "calibration_cpu_ns": 5969083.0, "calibration_gpu_ticks": 5969083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4757333.0, "compute_span_ns": 4757333.0, "command_buffer_ns": 4757291.63736105, "calibration_cpu_ns": 5837542.0, "calibration_gpu_ticks": 5837542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4873708.0, "compute_span_ns": 4873708.0, "command_buffer_ns": 4873708.356171846, "calibration_cpu_ns": 5830291.0, "calibration_gpu_ticks": 5830291.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 97250.0, "compute_span_ns": 97250.0, "command_buffer_ns": 97249.98380988836, "calibration_cpu_ns": 619292.0, "calibration_gpu_ticks": 619292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70709.0, "compute_span_ns": 70709.0, "command_buffer_ns": 70749.9566487968, "calibration_cpu_ns": 564167.0, "calibration_gpu_ticks": 564167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 94875.0, "compute_span_ns": 94875.0, "command_buffer_ns": 94874.99482929707, "calibration_cpu_ns": 614042.0, "calibration_gpu_ticks": 614042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70792.0, "compute_span_ns": 70792.0, "command_buffer_ns": 70833.3682268858, "calibration_cpu_ns": 508167.0, "calibration_gpu_ticks": 508167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 92375.0, "compute_span_ns": 92375.0, "command_buffer_ns": 92375.03400072455, "calibration_cpu_ns": 541584.0, "calibration_gpu_ticks": 541584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70750.0, "compute_span_ns": 70750.0, "command_buffer_ns": 70750.01485645771, "calibration_cpu_ns": 595333.0, "calibration_gpu_ticks": 595333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 460625.0, "compute_span_ns": 460625.0, "command_buffer_ns": 460625.04407018423, "calibration_cpu_ns": 1674750.0, "calibration_gpu_ticks": 1674750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 91708.0, "compute_span_ns": 91708.0, "command_buffer_ns": 91708.3234526217, "calibration_cpu_ns": 536292.0, "calibration_gpu_ticks": 536292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 494708.0, "compute_span_ns": 494708.0, "command_buffer_ns": 494666.6886098683, "calibration_cpu_ns": 1638750.0, "calibration_gpu_ticks": 1638750.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 72.7200625, 75.55403125, 69.613921875, 73.2545625, 70.596359375, 74.004546875, 74.197265625, 74.333328125, 76.1516875 ], "compute_throughput_us_p50": 74.004546875, "compute_throughput_us_p90": 75.67356249999999, "command_buffer_throughput_us": [ 72.72070251929108, 75.55403590231435, 69.61328108445741, 73.25520800804952, 70.59700510581024, 74.0039067750331, 74.19726625812473, 74.3326818337664, 76.1516930651851 ], "command_buffer_throughput_us_p50": 74.0039067750331, "command_buffer_throughput_us_p90": 75.6735673348885, "compute_latency_us": [ 97.25, 70.709, 94.875, 70.792, 92.375, 70.75, 460.625, 91.708, 494.708 ], "compute_latency_us_p50": 92.375, "compute_latency_us_p90": 467.4416, "command_buffer_latency_us": [ 97.24998380988836, 70.7499566487968, 94.87499482929707, 70.8333682268858, 92.37503400072455, 70.75001485645771, 460.62504407018423, 91.7083234526217, 494.6666886098683 ], "command_buffer_latency_us_p50": 92.37503400072455, "command_buffer_latency_us_p90": 467.43337297812104, "counter_control_throughput_ratio": 0.9743058014444202, "counter_control_latency_ratio": 0.9906644964786063 }, "download_ms": 0.754666, "allocation_upload_ms": 17.526959, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7481828776658174, "device_compute_slowdown": 0.8437156838547294, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.416, "warmup_ms": 200.107667, "repetitions": 288, "throughput_us": [ 106.42390277777777, 107.72048611111111, 99.8951111111111, 111.92100694444444, 111.39916319444445, 115.40480208333335, 111.81770833333333, 117.55265972222222, 90.18431597222222 ], "latency_us": [ 405.791, 343.625, 566.667, 336.5, 367.375, 4720.416, 525.583, 406.25, 294.66700000000003 ], "throughput_us_p50": 111.39916319444445, "throughput_us_p90": 115.83437361111112, "latency_us_p50": 405.791, "latency_us_p90": 1397.4168000000009, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4763583.303429186, "command_buffers": 1 }, { "command_buffer_ns": 6945624.947547913, "command_buffers": 1 }, { "command_buffer_ns": 6742750.003468245, "command_buffers": 1 }, { "command_buffer_ns": 6865874.980576336, "command_buffers": 1 }, { "command_buffer_ns": 4476666.683331132, "command_buffers": 1 }, { "command_buffer_ns": 4332249.9841451645, "command_buffers": 1 }, { "command_buffer_ns": 4350958.333816379, "command_buffers": 1 }, { "command_buffer_ns": 4633083.357475698, "command_buffers": 1 }, { "command_buffer_ns": 4484666.686039418, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 414916.66343063116, "command_buffers": 1 }, { "command_buffer_ns": 431416.67265444994, "command_buffers": 1 }, { "command_buffer_ns": 104125.00705569983, "command_buffers": 1 }, { "command_buffer_ns": 295499.9799840152, "command_buffers": 1 }, { "command_buffer_ns": 211875.01261010766, "command_buffers": 1 }, { "command_buffer_ns": 344750.0057518482, "command_buffers": 1 }, { "command_buffer_ns": 302750.03518909216, "command_buffers": 1 }, { "command_buffer_ns": 478125.00270083547, "command_buffers": 1 }, { "command_buffer_ns": 586249.9820068479, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 74.43098911608104, 108.52538980543613, 105.35546880419133, 107.27929657150526, 69.94791692704894, 67.6914060022682, 67.98372396588093, 72.39192746055778, 70.0729169693659 ], "command_buffer_throughput_us_p50": 72.39192746055778, "command_buffer_throughput_us_p90": 107.52851521829143, "command_buffer_latency_us": [ 414.91666343063116, 431.41667265444994, 104.12500705569983, 295.4999799840152, 211.87501261010766, 344.7500057518482, 302.75003518909216, 478.12500270083547, 586.2499820068479 ], "command_buffer_latency_us_p50": 344.7500057518482, "command_buffer_latency_us_p90": 499.74999856203794 }, "throughput": [ { "compute_ns": 4918000.0, "compute_span_ns": 4918000.0, "command_buffer_ns": 4918000.020552427, "calibration_cpu_ns": 6665750.0, "calibration_gpu_ticks": 6665750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5008917.0, "compute_span_ns": 5008917.0, "command_buffer_ns": 5008958.338294178, "calibration_cpu_ns": 7699208.0, "calibration_gpu_ticks": 7699208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4426416.0, "compute_span_ns": 4426416.0, "command_buffer_ns": 4426374.9732635915, "calibration_cpu_ns": 7657041.0, "calibration_gpu_ticks": 7657041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 6985667.0, "compute_span_ns": 6985667.0, "command_buffer_ns": 6985666.695982218, "calibration_cpu_ns": 7959459.0, "calibration_gpu_ticks": 7959459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4321083.0, "compute_span_ns": 4321083.0, "command_buffer_ns": 4321083.368267864, "calibration_cpu_ns": 5035416.0, "calibration_gpu_ticks": 5035416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4634333.0, "compute_span_ns": 4634333.0, "command_buffer_ns": 4634291.690308601, "calibration_cpu_ns": 5595084.0, "calibration_gpu_ticks": 5595084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4416958.0, "compute_span_ns": 4416958.0, "command_buffer_ns": 4416958.3125039935, "calibration_cpu_ns": 7574041.0, "calibration_gpu_ticks": 7574041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4786917.0, "compute_span_ns": 4786917.0, "command_buffer_ns": 4786958.335898817, "calibration_cpu_ns": 7402583.0, "calibration_gpu_ticks": 7402583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4438167.0, "compute_span_ns": 4438167.0, "command_buffer_ns": 4438208.357896656, "calibration_cpu_ns": 7381667.0, "calibration_gpu_ticks": 7381667.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 279125.0, "compute_span_ns": 279125.0, "command_buffer_ns": 279125.0008158386, "calibration_cpu_ns": 2369000.0, "calibration_gpu_ticks": 2369000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 571709.0, "compute_span_ns": 571709.0, "command_buffer_ns": 571749.9880120158, "calibration_cpu_ns": 2570542.0, "calibration_gpu_ticks": 2570542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 297709.0, "compute_span_ns": 297709.0, "command_buffer_ns": 297749.9971166253, "calibration_cpu_ns": 2101667.0, "calibration_gpu_ticks": 2101667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 338209.0, "compute_span_ns": 338209.0, "command_buffer_ns": 338250.0144653022, "calibration_cpu_ns": 2456958.0, "calibration_gpu_ticks": 2456958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 300541.0, "compute_span_ns": 300541.0, "command_buffer_ns": 300499.95984882116, "calibration_cpu_ns": 2084208.0, "calibration_gpu_ticks": 2084208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 337209.0, "compute_span_ns": 337209.0, "command_buffer_ns": 337250.00685080886, "calibration_cpu_ns": 2296417.0, "calibration_gpu_ticks": 2296417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 402708.0, "compute_span_ns": 402708.0, "command_buffer_ns": 402708.36325362325, "calibration_cpu_ns": 2553583.0, "calibration_gpu_ticks": 2553583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 551083.0, "compute_span_ns": 551083.0, "command_buffer_ns": 551083.2997970283, "calibration_cpu_ns": 2506208.0, "calibration_gpu_ticks": 2506208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 113667.0, "compute_span_ns": 113667.0, "command_buffer_ns": 113708.3163484931, "calibration_cpu_ns": 2848791.0, "calibration_gpu_ticks": 2848791.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 76.84375, 78.264328125, 69.16275, 109.151046875, 67.516921875, 72.411453125, 69.01496875, 74.795578125, 69.346359375 ], "compute_throughput_us_p50": 72.411453125, "compute_throughput_us_p90": 84.44167187500001, "command_buffer_throughput_us": [ 76.84375032113167, 78.26497403584654, 69.16210895724362, 109.15104212472215, 67.51692762918537, 72.41080766107189, 69.0149736328749, 74.79622399841901, 69.34700559213525 ], "command_buffer_throughput_us_p50": 72.41080766107189, "command_buffer_throughput_us_p90": 84.44218765362166, "compute_latency_us": [ 279.125, 571.709, 297.709, 338.209, 300.541, 337.209, 402.708, 551.083, 113.667 ], "compute_latency_us_p50": 337.209, "compute_latency_us_p90": 555.2081999999999, "command_buffer_latency_us": [ 279.1250008158386, 571.7499880120158, 297.7499971166253, 338.25001446530223, 300.49995984882116, 337.25000685080886, 402.70836325362325, 551.0832997970283, 113.7083163484931 ], "command_buffer_latency_us_p50": 337.25000685080886, "command_buffer_latency_us_p90": 555.2166374400258, "counter_control_throughput_ratio": 1.015169067047747, "counter_control_latency_ratio": 1.1525925159405292 }, "download_ms": 0.881667, "allocation_upload_ms": 2.52975, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-14fmthxw/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "310a51aaac34ebd04d17becc4645f77793138816428bb43603834b11ab1687fb", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 25.615417, "capture_ms": 0.061541, "compile_ms": 56.06375, "allocation_upload_ms": 6.61575, "cold_call_ms": 10.352541, "warmup_ms": 201.134625, "download_ms": 3.315541, "repetitions": 349, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 95.0686504298, 94.27352149, 94.8771489971, 94.0703180516, 94.5872722063, 94.915713467, 94.2836676218, 94.5908567335, 94.0756905444 ], "latency_us": [ 416.375, 264.042, 323.666, 381.042, 352.333, 2931.292, 518.625, 373.25, 363.042 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 72, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4434417, "compute_span_ns": 4434417, "calibration_cpu_ns": 5653750, "calibration_gpu_ticks": 5653750, "compute_passes": 1, "command_buffer_ns": 4434499.94782, "command_buffers": 2 }, { "compute_ns": 4522958, "compute_span_ns": 4522958, "calibration_cpu_ns": 5666666, "calibration_gpu_ticks": 5666666, "compute_passes": 1, "command_buffer_ns": 4523041.71631, "command_buffers": 2 }, { "compute_ns": 4552250, "compute_span_ns": 4552250, "calibration_cpu_ns": 5692500, "calibration_gpu_ticks": 5692500, "compute_passes": 1, "command_buffer_ns": 4552291.7062, "command_buffers": 2 }, { "compute_ns": 4568125, "compute_span_ns": 4568125, "calibration_cpu_ns": 5816250, "calibration_gpu_ticks": 5816250, "compute_passes": 1, "command_buffer_ns": 4568166.68157, "command_buffers": 2 }, { "compute_ns": 4485875, "compute_span_ns": 4485875, "calibration_cpu_ns": 5779375, "calibration_gpu_ticks": 5779375, "compute_passes": 1, "command_buffer_ns": 4485999.99072, "command_buffers": 2 }, { "compute_ns": 4588791, "compute_span_ns": 4588791, "calibration_cpu_ns": 5592250, "calibration_gpu_ticks": 5592250, "compute_passes": 1, "command_buffer_ns": 4588833.36978, "command_buffers": 2 }, { "compute_ns": 4492208, "compute_span_ns": 4492208, "calibration_cpu_ns": 5706458, "calibration_gpu_ticks": 5706458, "compute_passes": 1, "command_buffer_ns": 4492291.715, "command_buffers": 2 }, { "compute_ns": 4547959, "compute_span_ns": 4547959, "calibration_cpu_ns": 5812833, "calibration_gpu_ticks": 5812833, "compute_passes": 1, "command_buffer_ns": 4548083.29232, "command_buffers": 2 }, { "compute_ns": 4545166, "compute_span_ns": 4545166, "calibration_cpu_ns": 5697042, "calibration_gpu_ticks": 5697042, "compute_passes": 1, "command_buffer_ns": 4545208.35774, "command_buffers": 2 } ], "latency": [ { "compute_ns": 100333, "compute_span_ns": 100333, "calibration_cpu_ns": 752750, "calibration_gpu_ticks": 752750, "compute_passes": 1, "command_buffer_ns": 100416.596979, "command_buffers": 2 }, { "compute_ns": 64750, "compute_span_ns": 64750, "calibration_cpu_ns": 481833, "calibration_gpu_ticks": 481833, "compute_passes": 1, "command_buffer_ns": 64875.0574328, "command_buffers": 2 }, { "compute_ns": 65375, "compute_span_ns": 65375, "calibration_cpu_ns": 497542, "calibration_gpu_ticks": 497542, "compute_passes": 1, "command_buffer_ns": 65458.2981952, "command_buffers": 2 }, { "compute_ns": 253042, "compute_span_ns": 253042, "calibration_cpu_ns": 1619667, "calibration_gpu_ticks": 1619667, "compute_passes": 1, "command_buffer_ns": 253166.654147, "command_buffers": 2 }, { "compute_ns": 264750, "compute_span_ns": 264750, "calibration_cpu_ns": 1704834, "calibration_gpu_ticks": 1704834, "compute_passes": 1, "command_buffer_ns": 264791.655354, "command_buffers": 2 }, { "compute_ns": 101584, "compute_span_ns": 101584, "calibration_cpu_ns": 776625, "calibration_gpu_ticks": 776625, "compute_passes": 1, "command_buffer_ns": 101624.988019, "command_buffers": 2 }, { "compute_ns": 96459, "compute_span_ns": 96459, "calibration_cpu_ns": 542375, "calibration_gpu_ticks": 542375, "compute_passes": 1, "command_buffer_ns": 96583.3896771, "command_buffers": 2 }, { "compute_ns": 66209, "compute_span_ns": 66209, "calibration_cpu_ns": 539584, "calibration_gpu_ticks": 539584, "compute_passes": 1, "command_buffer_ns": 66333.275754, "command_buffers": 2 }, { "compute_ns": 65375, "compute_span_ns": 65375, "calibration_cpu_ns": 503791, "calibration_gpu_ticks": 503791, "compute_passes": 1, "command_buffer_ns": 65416.6797176, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4421333.31671, "command_buffers": 2 }, { "command_buffer_ns": 4536374.99595, "command_buffers": 2 }, { "command_buffer_ns": 4595625.03966, "command_buffers": 2 }, { "command_buffer_ns": 4598666.62277, "command_buffers": 2 }, { "command_buffer_ns": 4415458.35929, "command_buffers": 2 }, { "command_buffer_ns": 4546083.3353, "command_buffers": 2 }, { "command_buffer_ns": 4506625.00225, "command_buffers": 2 }, { "command_buffer_ns": 4546999.98954, "command_buffers": 2 }, { "command_buffer_ns": 4571541.67816, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 241208.355874, "command_buffers": 2 }, { "command_buffer_ns": 284874.986392, "command_buffers": 2 }, { "command_buffer_ns": 93791.6338444, "command_buffers": 2 }, { "command_buffer_ns": 102499.965578, "command_buffers": 2 }, { "command_buffer_ns": 65500.033088, "command_buffers": 2 }, { "command_buffer_ns": 65749.976784, "command_buffers": 2 }, { "command_buffer_ns": 319041.660987, "command_buffers": 2 }, { "command_buffer_ns": 349749.985617, "command_buffers": 2 }, { "command_buffer_ns": 98124.9613687, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 69.08333307359375, 70.88085931171875, 71.8066412446875, 71.85416598078126, 68.99153686390625, 71.03255211406251, 70.41601566015625, 71.04687483656251, 71.43033872125 ], "command_buffer_throughput_us_p50": 71.03255211406251, "command_buffer_throughput_us_p90": 71.81614619190626, "command_buffer_latency_us": [ 241.208355874, 284.874986392, 93.7916338444, 102.499965578, 65.500033088, 65.749976784, 319.041660987, 349.74998561700005, 98.1249613687 ], "command_buffer_latency_us_p50": 102.499965578, "command_buffer_latency_us_p90": 325.18332591300003 }, "compute_throughput_us": [ 69.287765625, 70.67121875, 71.12890625, 71.376953125, 70.091796875, 71.699859375, 70.19075, 71.061859375, 71.01821875 ], "compute_throughput_us_p50": 71.01821875, "compute_throughput_us_p90": 71.441534375, "command_buffer_throughput_us": [ 69.28906168468751, 70.67252681734375, 71.129557909375, 71.37760439953125, 70.093749855, 71.7005214028125, 70.192058046875, 71.0638014425, 71.0188805896875 ], "command_buffer_throughput_us_p50": 71.0188805896875, "command_buffer_throughput_us_p90": 71.4421878001875, "compute_latency_us": [ 100.333, 64.75, 65.375, 253.042, 264.75, 101.584, 96.459, 66.209, 65.375 ], "compute_latency_us_p50": 96.459, "compute_latency_us_p90": 255.3836, "command_buffer_latency_us": [ 100.41659697899999, 64.8750574328, 65.4582981952, 253.166654147, 264.791655354, 101.624988019, 96.5833896771, 66.333275754, 65.41667971759999 ], "command_buffer_latency_us_p50": 96.5833896771, "command_buffer_latency_us_p90": 255.4916543884, "counter_control_throughput_ratio": 0.9970608074394414, "counter_control_latency_ratio": 0.6666670621331493 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 94.5872722063, "throughput_us_p90": 94.94630085956, "latency_us_p50": 373.25, "latency_us_p90": 1001.1584000000005 }, "slowdown": 0.8490842255358801, "device_compute_slowdown": 0.9807594749880945, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-90v38nhn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "53243cd32bb4620fa91e934e3d500807c417fa70fc3ac1977dac07575a48be10", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.190917, "capture_ms": 0.079125, "compile_ms": 74.110209, "allocation_upload_ms": 2.097292, "cold_call_ms": 1.997291, "warmup_ms": 202.385542, "download_ms": 0.303584, "repetitions": 3938, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.93683341798, 7.70551879126, 7.06860512951, 7.62419578466, 6.83186262062, 7.45464068055, 6.82196952768, 6.81271152869, 7.50570289487 ], "latency_us": [ 283.917, 215.166, 217.75, 228.916, 217.834, 213.208, 206.333, 203.916, 223.417 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 761834, "compute_span_ns": 761834, "calibration_cpu_ns": 2253875, "calibration_gpu_ticks": 2253875, "compute_passes": 1, "command_buffer_ns": 761874.951422, "command_buffers": 2 }, { "compute_ns": 295916, "compute_span_ns": 295916, "calibration_cpu_ns": 746667, "calibration_gpu_ticks": 746667, "compute_passes": 1, "command_buffer_ns": 295958.307106, "command_buffers": 2 }, { "compute_ns": 336125, "compute_span_ns": 336125, "calibration_cpu_ns": 776542, "calibration_gpu_ticks": 776542, "compute_passes": 1, "command_buffer_ns": 336166.704074, "command_buffers": 2 }, { "compute_ns": 319084, "compute_span_ns": 319084, "calibration_cpu_ns": 741459, "calibration_gpu_ticks": 741459, "compute_passes": 1, "command_buffer_ns": 319166.632835, "command_buffers": 2 }, { "compute_ns": 345875, "compute_span_ns": 345875, "calibration_cpu_ns": 821708, "calibration_gpu_ticks": 821708, "compute_passes": 1, "command_buffer_ns": 345958.280377, "command_buffers": 2 }, { "compute_ns": 355417, "compute_span_ns": 355417, "calibration_cpu_ns": 776750, "calibration_gpu_ticks": 776750, "compute_passes": 1, "command_buffer_ns": 355541.706085, "command_buffers": 2 }, { "compute_ns": 321542, "compute_span_ns": 321542, "calibration_cpu_ns": 826166, "calibration_gpu_ticks": 826166, "compute_passes": 1, "command_buffer_ns": 321625.033394, "command_buffers": 2 }, { "compute_ns": 798916, "compute_span_ns": 798916, "calibration_cpu_ns": 2305875, "calibration_gpu_ticks": 2305875, "compute_passes": 1, "command_buffer_ns": 798958.353698, "command_buffers": 2 }, { "compute_ns": 1027041, "compute_span_ns": 1027041, "calibration_cpu_ns": 2223000, "calibration_gpu_ticks": 2223000, "compute_passes": 1, "command_buffer_ns": 1027083.37363, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7209, "compute_span_ns": 7209, "calibration_cpu_ns": 416000, "calibration_gpu_ticks": 416000, "compute_passes": 1, "command_buffer_ns": 7291.67368263, "command_buffers": 2 }, { "compute_ns": 7709, "compute_span_ns": 7709, "calibration_cpu_ns": 441542, "calibration_gpu_ticks": 441542, "compute_passes": 1, "command_buffer_ns": 7791.67748988, "command_buffers": 2 }, { "compute_ns": 7084, "compute_span_ns": 7084, "calibration_cpu_ns": 378375, "calibration_gpu_ticks": 378375, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 356958, "calibration_gpu_ticks": 356958, "compute_passes": 1, "command_buffer_ns": 7083.34846422, "command_buffers": 2 }, { "compute_ns": 31125, "compute_span_ns": 31125, "calibration_cpu_ns": 1176458, "calibration_gpu_ticks": 1176458, "compute_passes": 1, "command_buffer_ns": 31125.0332743, "command_buffers": 2 }, { "compute_ns": 8917, "compute_span_ns": 8917, "calibration_cpu_ns": 636250, "calibration_gpu_ticks": 636250, "compute_passes": 1, "command_buffer_ns": 9083.30548555, "command_buffers": 2 }, { "compute_ns": 7709, "compute_span_ns": 7709, "calibration_cpu_ns": 457875, "calibration_gpu_ticks": 457875, "compute_passes": 1, "command_buffer_ns": 7750.00080466, "command_buffers": 2 }, { "compute_ns": 7792, "compute_span_ns": 7792, "calibration_cpu_ns": 372791, "calibration_gpu_ticks": 372791, "compute_passes": 1, "command_buffer_ns": 7833.29596743, "command_buffers": 2 }, { "compute_ns": 7375, "compute_span_ns": 7375, "calibration_cpu_ns": 416708, "calibration_gpu_ticks": 416708, "compute_passes": 1, "command_buffer_ns": 7416.64553061, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 494708.365295, "command_buffers": 2 }, { "command_buffer_ns": 628375.040833, "command_buffers": 2 }, { "command_buffer_ns": 998875.009827, "command_buffers": 2 }, { "command_buffer_ns": 313125.026878, "command_buffers": 2 }, { "command_buffer_ns": 1032958.33105, "command_buffers": 2 }, { "command_buffer_ns": 843624.99183, "command_buffers": 2 }, { "command_buffer_ns": 308624.992613, "command_buffers": 2 }, { "command_buffer_ns": 330333.30692, "command_buffers": 2 }, { "command_buffer_ns": 357666.693162, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7333.35036784, "command_buffers": 2 }, { "command_buffer_ns": 61708.2696408, "command_buffers": 2 }, { "command_buffer_ns": 9124.98217076, "command_buffers": 2 }, { "command_buffer_ns": 11749.973055, "command_buffers": 2 }, { "command_buffer_ns": 7499.94069338, "command_buffers": 2 }, { "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "command_buffer_ns": 7416.64553061, "command_buffers": 2 }, { "command_buffer_ns": 116666.662507, "command_buffers": 2 }, { "command_buffer_ns": 9291.68891162, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 7.729818207734375, 9.818360013015624, 15.607422028546875, 4.89257854496875, 16.13997392265625, 13.18164049734375, 4.822265509578124, 5.161457920625, 5.58854208065625 ], "command_buffer_throughput_us_p50": 7.729818207734375, "command_buffer_throughput_us_p90": 15.71393240736875, "command_buffer_latency_us": [ 7.3333503678400005, 61.7082696408, 9.124982170760001, 11.749973055, 7.49994069338, 7.1249669417699995, 7.416645530609999, 116.666662507, 9.29168891162 ], "command_buffer_latency_us_p50": 9.124982170760001, "command_buffer_latency_us_p90": 72.69994821404 }, "compute_throughput_us": [ 11.90365625, 4.6236875, 5.251953125, 4.9856875, 5.404296875, 5.553390625, 5.02409375, 12.4830625, 16.047515625 ], "compute_throughput_us_p50": 5.404296875, "compute_throughput_us_p90": 13.195953125, "command_buffer_throughput_us": [ 11.90429611596875, 4.62434854853125, 5.25260475115625, 4.986978638046875, 5.405598130890625, 5.555339157578125, 5.02539114678125, 12.48372427653125, 16.04817771296875 ], "command_buffer_throughput_us_p50": 5.405598130890625, "command_buffer_throughput_us_p90": 13.196614963818751, "compute_latency_us": [ 7.209, 7.709, 7.084, 7.042, 31.125, 8.917, 7.709, 7.792, 7.375 ], "compute_latency_us_p50": 7.709, "compute_latency_us_p90": 13.358600000000003, "command_buffer_latency_us": [ 7.29167368263, 7.7916774898800005, 7.208320312200001, 7.08334846422, 31.1250332743, 9.08330548555, 7.75000080466, 7.83329596743, 7.416645530609999 ], "command_buffer_latency_us_p50": 7.75000080466, "command_buffer_latency_us_p90": 13.491651043300003, "counter_control_throughput_ratio": 1.019294548306271, "counter_control_latency_ratio": 0.7982020923381207 }, "correctness": { "max_abs_error": 2.1991937471099732e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 7.06860512951, "throughput_us_p90": 7.64046038598, "latency_us_p50": 217.75, "latency_us_p90": 239.9162 }, "torch": { "cold_call_ms": 9.236125, "warmup_ms": 200.471042, "repetitions": 1753, "throughput_us": [ 15.378969195664576, 16.9092983456931, 14.536413576725614, 14.411936679977181, 14.975066172276097, 14.31600570450656, 14.345003993154593, 14.880205362236167, 14.01851568739304 ], "latency_us": [ 303.66700000000003, 300.333, 227.625, 227.5, 240.542, 224.291, 242.209, 2148.459, 619.958 ], "throughput_us_p50": 14.536413576725614, "throughput_us_p90": 15.685035025670281, "latency_us_p50": 242.209, "latency_us_p90": 925.6582000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 526499.9927021563, "command_buffers": 1 }, { "command_buffer_ns": 535124.9710656703, "command_buffers": 1 }, { "command_buffer_ns": 484624.99398738146, "command_buffers": 1 }, { "command_buffer_ns": 505083.3569839597, "command_buffers": 1 }, { "command_buffer_ns": 832375.0225827098, "command_buffers": 1 }, { "command_buffer_ns": 851958.3498127759, "command_buffers": 1 }, { "command_buffer_ns": 520250.00331923366, "command_buffers": 1 }, { "command_buffer_ns": 485499.9715462327, "command_buffers": 1 }, { "command_buffer_ns": 727708.3350345492, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10041.636414825916, "command_buffers": 1 }, { "command_buffer_ns": 10583.316907286644, "command_buffers": 1 }, { "command_buffer_ns": 14166.638720780611, "command_buffers": 1 }, { "command_buffer_ns": 9874.98788163066, "command_buffers": 1 }, { "command_buffer_ns": 198708.3232961595, "command_buffers": 1 }, { "command_buffer_ns": 11374.999303370714, "command_buffers": 1 }, { "command_buffer_ns": 10374.991688877344, "command_buffers": 1 }, { "command_buffer_ns": 195708.3586603403, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 8.226562385971192, 8.361327672901098, 7.572265531052835, 7.89192745287437, 13.00585972785484, 13.311849215824623, 8.128906301863026, 7.585937055409886, 11.370442734914832 ], "command_buffer_throughput_us_p50": 8.226562385971192, "command_buffer_throughput_us_p90": 13.067057625448797, "command_buffer_latency_us": [ 10.25001984089613, 10.041636414825916, 10.583316907286644, 14.166638720780611, 9.874987881630659, 198.7083232961595, 11.374999303370714, 10.374991688877344, 195.7083586603403 ], "command_buffer_latency_us_p50": 10.583316907286644, "command_buffer_latency_us_p90": 196.30835158750415 }, "throughput": [ { "compute_ns": 506000.0, "compute_span_ns": 506000.0, "command_buffer_ns": 505999.95302036405, "calibration_cpu_ns": 1209833.0, "calibration_gpu_ticks": 1209833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 938916.0, "compute_span_ns": 938916.0, "command_buffer_ns": 938875.0186190009, "calibration_cpu_ns": 1918000.0, "calibration_gpu_ticks": 1918000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 941166.0, "compute_span_ns": 941166.0, "command_buffer_ns": 941124.9775439501, "calibration_cpu_ns": 2031167.0, "calibration_gpu_ticks": 2031167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 543209.0, "compute_span_ns": 543209.0, "command_buffer_ns": 543250.0038295984, "calibration_cpu_ns": 1286917.0, "calibration_gpu_ticks": 1286917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 505000.0, "compute_span_ns": 505000.0, "command_buffer_ns": 505000.0036135316, "calibration_cpu_ns": 1263042.0, "calibration_gpu_ticks": 1263042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 486708.0, "compute_span_ns": 486708.0, "command_buffer_ns": 486666.68590158224, "calibration_cpu_ns": 1179750.0, "calibration_gpu_ticks": 1179750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 820583.0, "compute_span_ns": 820583.0, "command_buffer_ns": 820541.6379496455, "calibration_cpu_ns": 2307125.0, "calibration_gpu_ticks": 2307125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 541291.0, "compute_span_ns": 541291.0, "command_buffer_ns": 541249.9886006117, "calibration_cpu_ns": 1226291.0, "calibration_gpu_ticks": 1226291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 527500.0, "compute_span_ns": 527500.0, "command_buffer_ns": 527500.0003166497, "calibration_cpu_ns": 1325416.0, "calibration_gpu_ticks": 1325416.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 202875.0, "compute_span_ns": 202875.0, "command_buffer_ns": 202875.00228732824, "calibration_cpu_ns": 1379375.0, "calibration_gpu_ticks": 1379375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11125.0, "compute_span_ns": 11125.0, "command_buffer_ns": 11124.997399747372, "calibration_cpu_ns": 687875.0, "calibration_gpu_ticks": 687875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10459.0, "compute_span_ns": 10459.0, "command_buffer_ns": 10499.963536858559, "calibration_cpu_ns": 438375.0, "calibration_gpu_ticks": 438375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 189125.0, "compute_span_ns": 189125.0, "command_buffer_ns": 189125.01400336623, "calibration_cpu_ns": 1275042.0, "calibration_gpu_ticks": 1275042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10625.0, "compute_span_ns": 10625.0, "command_buffer_ns": 10624.993592500687, "calibration_cpu_ns": 437917.0, "calibration_gpu_ticks": 437917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10458.0, "compute_span_ns": 10458.0, "command_buffer_ns": 10416.668374091387, "calibration_cpu_ns": 479625.0, "calibration_gpu_ticks": 479625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10042.0, "compute_span_ns": 10042.0, "command_buffer_ns": 10041.69462248683, "calibration_cpu_ns": 452208.0, "calibration_gpu_ticks": 452208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 14583.0, "compute_span_ns": 14583.0, "command_buffer_ns": 14541.670680046082, "calibration_cpu_ns": 498250.0, "calibration_gpu_ticks": 498250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11125.0, "compute_span_ns": 11125.0, "command_buffer_ns": 11124.997399747372, "calibration_cpu_ns": 649416.0, "calibration_gpu_ticks": 649416.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 7.90625, 14.6705625, 14.70571875, 8.487640625, 7.890625, 7.6048125, 12.821609375, 8.457671875, 8.2421875 ], "compute_throughput_us_p50": 8.457671875, "compute_throughput_us_p90": 14.677593750000002, "command_buffer_throughput_us": [ 7.906249265943188, 14.66992216592189, 14.70507777412422, 8.488281309837475, 7.890625056461431, 7.6041669672122225, 12.820963092963211, 8.457031071884558, 8.242187504947651 ], "command_buffer_throughput_us_p50": 8.457031071884558, "command_buffer_throughput_us_p90": 14.676953287562355, "compute_latency_us": [ 202.875, 11.125, 10.459, 189.125, 10.625, 10.458, 10.042, 14.583, 11.125 ], "compute_latency_us_p50": 11.125, "compute_latency_us_p90": 191.875, "command_buffer_latency_us": [ 202.87500228732824, 11.124997399747372, 10.499963536858559, 189.12501400336623, 10.624993592500687, 10.416668374091387, 10.04169462248683, 14.541670680046082, 11.124997399747372 ], "command_buffer_latency_us_p50": 11.124997399747372, "command_buffer_latency_us_p90": 191.87501166015863, "counter_control_throughput_ratio": 1.0755650454878298, "counter_control_latency_ratio": 1.075950038608673 }, "download_ms": 0.315042, "allocation_upload_ms": 0.851292, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.48626885113035095, "device_compute_slowdown": 0.6389816198680561, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.265542, "warmup_ms": 200.2815, "repetitions": 1383, "throughput_us": [ 14.016569775849604, 16.589630513376715, 16.35635068691251, 14.916003615328995, 14.813388286334057, 15.167088937093276, 14.996565437454809, 15.210292118582792, 15.666907447577728 ], "latency_us": [ 318.5, 267.666, 446.83299999999997, 302.08299999999997, 239.958, 220.208, 231.333, 232.917, 166.042 ], "throughput_us_p50": 15.167088937093276, "throughput_us_p90": 16.403006652205352, "latency_us_p50": 239.958, "latency_us_p90": 344.1666, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 763749.99480322, "command_buffers": 1 }, { "command_buffer_ns": 771666.644141078, "command_buffers": 1 }, { "command_buffer_ns": 570749.9803975224, "command_buffers": 1 }, { "command_buffer_ns": 517499.9823793769, "command_buffers": 1 }, { "command_buffer_ns": 491166.6619591415, "command_buffers": 1 }, { "command_buffer_ns": 485291.7045354843, "command_buffers": 1 }, { "command_buffer_ns": 931374.9615103006, "command_buffers": 1 }, { "command_buffer_ns": 854958.372656256, "command_buffers": 1 }, { "command_buffer_ns": 560166.6634902358, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 249041.65184125304, "command_buffers": 1 }, { "command_buffer_ns": 9833.311196416616, "command_buffers": 1 }, { "command_buffer_ns": 10166.666470468044, "command_buffers": 1 }, { "command_buffer_ns": 10791.700333356857, "command_buffers": 1 }, { "command_buffer_ns": 9874.98788163066, "command_buffers": 1 }, { "command_buffer_ns": 216624.99057129025, "command_buffers": 1 }, { "command_buffer_ns": 11291.704140603542, "command_buffers": 1 }, { "command_buffer_ns": 10374.991688877344, "command_buffers": 1 }, { "command_buffer_ns": 221125.02483651042, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 11.933593668800313, 12.057291314704344, 8.917968443711288, 8.085937224677764, 7.674479093111586, 7.582682883366942, 14.552733773598447, 13.358724572754, 8.752604117034934 ], "command_buffer_throughput_us_p50": 8.917968443711288, "command_buffer_throughput_us_p90": 13.597526412922889, "command_buffer_latency_us": [ 249.04165184125304, 9.833311196416616, 10.166666470468044, 10.791700333356857, 9.874987881630659, 216.62499057129025, 11.291704140603542, 10.374991688877344, 221.12502483651042 ], "command_buffer_latency_us_p50": 10.791700333356857, "command_buffer_latency_us_p90": 226.70835023745894 }, "throughput": [ { "compute_ns": 562250.0, "compute_span_ns": 562250.0, "command_buffer_ns": 562249.9738819897, "calibration_cpu_ns": 1296000.0, "calibration_gpu_ticks": 1296000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483667.0, "compute_span_ns": 483667.0, "command_buffer_ns": 483666.66305810213, "calibration_cpu_ns": 1169500.0, "calibration_gpu_ticks": 1169500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 482959.0, "compute_span_ns": 482959.0, "command_buffer_ns": 483000.0107176602, "calibration_cpu_ns": 1169125.0, "calibration_gpu_ticks": 1169125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 865667.0, "compute_span_ns": 865667.0, "command_buffer_ns": 865708.3380967379, "calibration_cpu_ns": 2168792.0, "calibration_gpu_ticks": 2168792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 834916.0, "compute_span_ns": 834916.0, "command_buffer_ns": 834875.0416189432, "calibration_cpu_ns": 2127833.0, "calibration_gpu_ticks": 2127833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 557666.0, "compute_span_ns": 557666.0, "command_buffer_ns": 557625.0259764493, "calibration_cpu_ns": 1324541.0, "calibration_gpu_ticks": 1324541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 548125.0, "compute_span_ns": 548125.0, "command_buffer_ns": 548125.0118464231, "calibration_cpu_ns": 1290500.0, "calibration_gpu_ticks": 1290500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483458.0, "compute_span_ns": 483458.0, "command_buffer_ns": 483416.6611544788, "calibration_cpu_ns": 1158000.0, "calibration_gpu_ticks": 1158000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483125.0, "compute_span_ns": 483125.0, "command_buffer_ns": 483124.9825656414, "calibration_cpu_ns": 1222417.0, "calibration_gpu_ticks": 1222417.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 11250.0, "compute_span_ns": 11250.0, "command_buffer_ns": 11250.0274553895, "calibration_cpu_ns": 506292.0, "calibration_gpu_ticks": 506292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9792.0, "compute_span_ns": 9792.0, "command_buffer_ns": 9833.36940407753, "calibration_cpu_ns": 388458.0, "calibration_gpu_ticks": 388458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10125.0, "compute_span_ns": 10125.0, "command_buffer_ns": 10124.989785254002, "calibration_cpu_ns": 391625.0, "calibration_gpu_ticks": 391625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 169750.0, "compute_span_ns": 169750.0, "command_buffer_ns": 169750.01199170947, "calibration_cpu_ns": 1524250.0, "calibration_gpu_ticks": 1524250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10375.0, "compute_span_ns": 10375.0, "command_buffer_ns": 10375.049896538258, "calibration_cpu_ns": 414667.0, "calibration_gpu_ticks": 414667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10375.0, "compute_span_ns": 10375.0, "command_buffer_ns": 10374.991688877344, "calibration_cpu_ns": 412416.0, "calibration_gpu_ticks": 412416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10042.0, "compute_span_ns": 10042.0, "command_buffer_ns": 10083.371307700872, "calibration_cpu_ns": 416125.0, "calibration_gpu_ticks": 416125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10167.0, "compute_span_ns": 10167.0, "command_buffer_ns": 10208.343155682087, "calibration_cpu_ns": 500667.0, "calibration_gpu_ticks": 500667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11792.0, "compute_span_ns": 11792.0, "command_buffer_ns": 11833.326425403357, "calibration_cpu_ns": 687500.0, "calibration_gpu_ticks": 687500.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 8.78515625, 7.557296875, 7.546234375, 13.526046875, 13.0455625, 8.71353125, 8.564453125, 7.55403125, 7.548828125 ], "compute_throughput_us_p50": 8.564453125, "compute_throughput_us_p90": 13.141659375000001, "command_buffer_throughput_us": [ 8.78515584190609, 7.557291610282846, 7.5468751674634404, 13.526692782761529, 13.044922525295988, 8.71289103088202, 8.564453310100362, 7.553385330538731, 7.548827852588147 ], "command_buffer_throughput_us_p50": 8.564453310100362, "command_buffer_throughput_us_p90": 13.141276576789096, "compute_latency_us": [ 11.25, 9.792, 10.125, 169.75, 10.375, 10.375, 10.042, 10.167, 11.792 ], "compute_latency_us_p50": 10.375, "compute_latency_us_p90": 43.38360000000003, "command_buffer_latency_us": [ 11.2500274553895, 9.83336940407753, 10.124989785254002, 169.75001199170947, 10.375049896538258, 10.374991688877344, 10.083371307700872, 10.208343155682087, 11.833326425403357 ], "command_buffer_latency_us_p50": 10.374991688877344, "command_buffer_latency_us_p90": 43.41666353866461, "counter_control_throughput_ratio": 0.846254975569609, "counter_control_latency_ratio": 0.983937477909123 }, "download_ms": 0.3705, "allocation_upload_ms": 0.673083, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-3w2f7p_7/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "90a0b6dbe0fbfeffc09b3a8db9fc5856f9e2631dc2ce0e6d9ee991918165a6cf", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.028625, "capture_ms": 0.089292, "compile_ms": 65.243416, "allocation_upload_ms": 2.088958, "cold_call_ms": 2.124916, "warmup_ms": 200.136333, "download_ms": 0.269958, "repetitions": 3229, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.0525447507, 6.80435119232, 6.01858160421, 7.19554567978, 6.09648250232, 6.8158872716, 6.09343728709, 6.8267138433, 6.06743573862 ], "latency_us": [ 1870.333, 840.583, 274.125, 244.875, 204.708, 191.167, 178.583, 185.458, 188.084 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 293666, "compute_span_ns": 293666, "calibration_cpu_ns": 775416, "calibration_gpu_ticks": 775416, "compute_passes": 1, "command_buffer_ns": 293666.613288, "command_buffers": 2 }, { "compute_ns": 270583, "compute_span_ns": 270583, "calibration_cpu_ns": 785750, "calibration_gpu_ticks": 785750, "compute_passes": 1, "command_buffer_ns": 270583.317615, "command_buffers": 2 }, { "compute_ns": 297542, "compute_span_ns": 297542, "calibration_cpu_ns": 803500, "calibration_gpu_ticks": 803500, "compute_passes": 1, "command_buffer_ns": 297666.701954, "command_buffers": 2 }, { "compute_ns": 808125, "compute_span_ns": 808125, "calibration_cpu_ns": 2077667, "calibration_gpu_ticks": 2077667, "compute_passes": 1, "command_buffer_ns": 808208.365925, "command_buffers": 2 }, { "compute_ns": 941583, "compute_span_ns": 941583, "calibration_cpu_ns": 2365000, "calibration_gpu_ticks": 2365000, "compute_passes": 1, "command_buffer_ns": 941708.334722, "command_buffers": 2 }, { "compute_ns": 297458, "compute_span_ns": 297458, "calibration_cpu_ns": 785750, "calibration_gpu_ticks": 785750, "compute_passes": 1, "command_buffer_ns": 297541.730106, "command_buffers": 2 }, { "compute_ns": 270750, "compute_span_ns": 270750, "calibration_cpu_ns": 758583, "calibration_gpu_ticks": 758583, "compute_passes": 1, "command_buffer_ns": 270791.642834, "command_buffers": 2 }, { "compute_ns": 283459, "compute_span_ns": 283459, "calibration_cpu_ns": 814125, "calibration_gpu_ticks": 814125, "compute_passes": 1, "command_buffer_ns": 283583.300188, "command_buffers": 2 }, { "compute_ns": 271667, "compute_span_ns": 271667, "calibration_cpu_ns": 860792, "calibration_gpu_ticks": 860792, "compute_passes": 1, "command_buffer_ns": 271666.6786, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7750, "compute_span_ns": 7750, "calibration_cpu_ns": 707333, "calibration_gpu_ticks": 707333, "compute_passes": 1, "command_buffer_ns": 7791.67748988, "command_buffers": 2 }, { "compute_ns": 6917, "compute_span_ns": 6917, "calibration_cpu_ns": 396333, "calibration_gpu_ticks": 396333, "compute_passes": 1, "command_buffer_ns": 6958.31840858, "command_buffers": 2 }, { "compute_ns": 139667, "compute_span_ns": 139667, "calibration_cpu_ns": 1456000, "calibration_gpu_ticks": 1456000, "compute_passes": 1, "command_buffer_ns": 139666.663017, "command_buffers": 2 }, { "compute_ns": 9083, "compute_span_ns": 9083, "calibration_cpu_ns": 584250, "calibration_gpu_ticks": 584250, "compute_passes": 1, "command_buffer_ns": 9208.27733353, "command_buffers": 2 }, { "compute_ns": 7958, "compute_span_ns": 7958, "calibration_cpu_ns": 531666, "calibration_gpu_ticks": 531666, "compute_passes": 1, "command_buffer_ns": 8000.00270829, "command_buffers": 2 }, { "compute_ns": 49958, "compute_span_ns": 49958, "calibration_cpu_ns": 966083, "calibration_gpu_ticks": 966083, "compute_passes": 1, "command_buffer_ns": 49958.3547935, "command_buffers": 2 }, { "compute_ns": 6917, "compute_span_ns": 6917, "calibration_cpu_ns": 390333, "calibration_gpu_ticks": 390333, "compute_passes": 1, "command_buffer_ns": 6916.64172336, "command_buffers": 2 }, { "compute_ns": 7292, "compute_span_ns": 7292, "calibration_cpu_ns": 470958, "calibration_gpu_ticks": 470958, "compute_passes": 1, "command_buffer_ns": 7333.35036784, "command_buffers": 2 }, { "compute_ns": 30792, "compute_span_ns": 30792, "calibration_cpu_ns": 550750, "calibration_gpu_ticks": 550750, "compute_passes": 1, "command_buffer_ns": 30791.6780002, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 957166.659646, "command_buffers": 2 }, { "command_buffer_ns": 975750.037469, "command_buffers": 2 }, { "command_buffer_ns": 295208.301395, "command_buffers": 2 }, { "command_buffer_ns": 292541.692033, "command_buffers": 2 }, { "command_buffer_ns": 281333.341263, "command_buffers": 2 }, { "command_buffer_ns": 287916.68592, "command_buffers": 2 }, { "command_buffer_ns": 921583.385207, "command_buffers": 2 }, { "command_buffer_ns": 934374.984354, "command_buffers": 2 }, { "command_buffer_ns": 270833.319519, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 97500.0439212, "command_buffers": 2 }, { "command_buffer_ns": 15458.3831318, "command_buffers": 2 }, { "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "command_buffer_ns": 6874.96503815, "command_buffers": 2 }, { "command_buffer_ns": 7374.9688454, "command_buffers": 2 }, { "command_buffer_ns": 42083.3239332, "command_buffers": 2 }, { "command_buffer_ns": 7416.64553061, "command_buffers": 2 }, { "command_buffer_ns": 14458.3755173, "command_buffers": 2 }, { "command_buffer_ns": 10499.9635369, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 14.95572905696875, 15.246094335453124, 4.6126297092968755, 4.570963938015625, 4.3958334572343745, 4.4986982175, 14.399740393859375, 14.599609130531249, 4.231770617484375 ], "command_buffer_throughput_us_p50": 4.6126297092968755, "command_buffer_throughput_us_p90": 15.013802112665624, "command_buffer_latency_us": [ 97.5000439212, 15.4583831318, 7.45832221583, 6.87496503815, 7.3749688454, 42.0833239332, 7.416645530609999, 14.4583755173, 10.499963536900001 ], "command_buffer_latency_us_p50": 10.499963536900001, "command_buffer_latency_us_p90": 53.16666793080001 }, "compute_throughput_us": [ 4.58853125, 4.227859375, 4.64909375, 12.626953125, 14.712234375, 4.64778125, 4.23046875, 4.429046875, 4.244796875 ], "compute_throughput_us_p50": 4.58853125, "compute_throughput_us_p90": 13.044009375, "command_buffer_throughput_us": [ 4.588540832625, 4.227864337734375, 4.651042218031249, 12.628255717578126, 14.71419273003125, 4.64908953290625, 4.23111941928125, 4.4309890654375, 4.244791853125 ], "command_buffer_throughput_us_p50": 4.588540832625, "command_buffer_throughput_us_p90": 13.045443120068752, "compute_latency_us": [ 7.75, 6.917, 139.667, 9.083, 7.958, 49.958, 6.917, 7.292, 30.792 ], "compute_latency_us_p50": 7.958, "compute_latency_us_p90": 67.89980000000001, "command_buffer_latency_us": [ 7.7916774898800005, 6.95831840858, 139.66666301700002, 9.20827733353, 8.00000270829, 49.9583547935, 6.91664172336, 7.3333503678400005, 30.7916780002 ], "command_buffer_latency_us_p50": 8.00000270829, "command_buffer_latency_us_p90": 67.90001643820003, "counter_control_throughput_ratio": 1.0030770183021793, "counter_control_latency_ratio": 1.0847507123070566 }, "correctness": { "max_abs_error": 2.43199801630567e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 6.09648250232, "throughput_us_p90": 6.900480210596, "latency_us_p50": 204.708, "latency_us_p90": 1046.5330000000001 }, "slowdown": 0.40195468804894946, "device_compute_slowdown": 0.5357646522234891, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-sj60hv22/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "83a70ba0e4d1dab6671da56395e0b13750f8d9c9277e4782c4e371450de131c7", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.439417, "capture_ms": 0.078334, "compile_ms": 64.86825, "allocation_upload_ms": 2.557667, "cold_call_ms": 6.29, "warmup_ms": 200.508917, "download_ms": 0.920708, "repetitions": 1213, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 22.9824130256, 24.4746158285, 22.6265111294, 24.7409661995, 22.6782423743, 23.7644608409, 23.1671813685, 22.6654295136, 24.8410626546 ], "latency_us": [ 286.916, 265.541, 247.5, 234.083, 254.459, 282, 235.542, 273.709, 177.75 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1695958, "compute_span_ns": 1695958, "calibration_cpu_ns": 2614042, "calibration_gpu_ticks": 2614042, "compute_passes": 1, "command_buffer_ns": 1696000.05029, "command_buffers": 2 }, { "compute_ns": 1037834, "compute_span_ns": 1037834, "calibration_cpu_ns": 1481875, "calibration_gpu_ticks": 1481875, "compute_passes": 1, "command_buffer_ns": 1037874.95755, "command_buffers": 2 }, { "compute_ns": 1771458, "compute_span_ns": 1771458, "calibration_cpu_ns": 2919333, "calibration_gpu_ticks": 2919333, "compute_passes": 1, "command_buffer_ns": 1771458.36642, "command_buffers": 2 }, { "compute_ns": 1063625, "compute_span_ns": 1063625, "calibration_cpu_ns": 1483000, "calibration_gpu_ticks": 1483000, "compute_passes": 1, "command_buffer_ns": 1063624.979, "command_buffers": 2 }, { "compute_ns": 1613125, "compute_span_ns": 1613125, "calibration_cpu_ns": 2532500, "calibration_gpu_ticks": 2532500, "compute_passes": 1, "command_buffer_ns": 1613166.64889, "command_buffers": 2 }, { "compute_ns": 1031750, "compute_span_ns": 1031750, "calibration_cpu_ns": 1490583, "calibration_gpu_ticks": 1490583, "compute_passes": 1, "command_buffer_ns": 1031833.29338, "command_buffers": 2 }, { "compute_ns": 1677625, "compute_span_ns": 1677625, "calibration_cpu_ns": 2662417, "calibration_gpu_ticks": 2662417, "compute_passes": 1, "command_buffer_ns": 1677625.05589, "command_buffers": 2 }, { "compute_ns": 1037458, "compute_span_ns": 1037458, "calibration_cpu_ns": 1615667, "calibration_gpu_ticks": 1615667, "compute_passes": 1, "command_buffer_ns": 1037458.30711, "command_buffers": 2 }, { "compute_ns": 1600292, "compute_span_ns": 1600292, "calibration_cpu_ns": 2450458, "calibration_gpu_ticks": 2450458, "compute_passes": 1, "command_buffer_ns": 1600333.37306, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18000, "compute_span_ns": 18000, "calibration_cpu_ns": 417583, "calibration_gpu_ticks": 417583, "compute_passes": 1, "command_buffer_ns": 18083.3158083, "command_buffers": 2 }, { "compute_ns": 153458, "compute_span_ns": 153458, "calibration_cpu_ns": 1524541, "calibration_gpu_ticks": 1524541, "compute_passes": 1, "command_buffer_ns": 153541.623149, "command_buffers": 2 }, { "compute_ns": 19916, "compute_span_ns": 19916, "calibration_cpu_ns": 457959, "calibration_gpu_ticks": 457959, "compute_passes": 1, "command_buffer_ns": 19875.0058189, "command_buffers": 2 }, { "compute_ns": 20042, "compute_span_ns": 20042, "calibration_cpu_ns": 427209, "calibration_gpu_ticks": 427209, "compute_passes": 1, "command_buffer_ns": 20041.6543521, "command_buffers": 2 }, { "compute_ns": 18125, "compute_span_ns": 18125, "calibration_cpu_ns": 464542, "calibration_gpu_ticks": 464542, "compute_passes": 1, "command_buffer_ns": 18208.345864, "command_buffers": 2 }, { "compute_ns": 17916, "compute_span_ns": 17916, "calibration_cpu_ns": 512125, "calibration_gpu_ticks": 512125, "compute_passes": 1, "command_buffer_ns": 17958.2857527, "command_buffers": 2 }, { "compute_ns": 17916, "compute_span_ns": 17916, "calibration_cpu_ns": 485250, "calibration_gpu_ticks": 485250, "compute_passes": 1, "command_buffer_ns": 17958.2857527, "command_buffers": 2 }, { "compute_ns": 20000, "compute_span_ns": 20000, "calibration_cpu_ns": 476334, "calibration_gpu_ticks": 476334, "compute_passes": 1, "command_buffer_ns": 20041.6543521, "command_buffers": 2 }, { "compute_ns": 17958, "compute_span_ns": 17958, "calibration_cpu_ns": 564458, "calibration_gpu_ticks": 564458, "compute_passes": 1, "command_buffer_ns": 17958.3439603, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1683333.30657, "command_buffers": 2 }, { "command_buffer_ns": 1731041.58603, "command_buffers": 2 }, { "command_buffer_ns": 1066083.26314, "command_buffers": 2 }, { "command_buffer_ns": 1701041.70684, "command_buffers": 2 }, { "command_buffer_ns": 1029125.00734, "command_buffers": 2 }, { "command_buffer_ns": 1545708.33547, "command_buffers": 2 }, { "command_buffer_ns": 1033791.63193, "command_buffers": 2 }, { "command_buffer_ns": 1627583.34773, "command_buffers": 2 }, { "command_buffer_ns": 1038249.98951, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 23458.3858401, "command_buffers": 2 }, { "command_buffer_ns": 22833.2937695, "command_buffers": 2 }, { "command_buffer_ns": 18208.4040716, "command_buffers": 2 }, { "command_buffer_ns": 213166.640606, "command_buffers": 2 }, { "command_buffer_ns": 25583.372917, "command_buffers": 2 }, { "command_buffer_ns": 258583.342656, "command_buffers": 2 }, { "command_buffer_ns": 24874.9856837, "command_buffers": 2 }, { "command_buffer_ns": 490208.389238, "command_buffers": 2 }, { "command_buffer_ns": 21916.6395254, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 26.30208291515625, 27.04752478171875, 16.657550986562498, 26.578776669375003, 16.0800782396875, 24.15169274171875, 16.15299424890625, 25.43098980828125, 16.22265608609375 ], "command_buffer_throughput_us_p50": 24.15169274171875, "command_buffer_throughput_us_p90": 26.672526291843752, "command_buffer_latency_us": [ 23.4583858401, 22.8332937695, 18.2084040716, 213.166640606, 25.583372917000002, 258.583342656, 24.8749856837, 490.208389238, 21.9166395254 ], "command_buffer_latency_us_p50": 24.8749856837, "command_buffer_latency_us_p90": 304.90835197240006 }, "compute_throughput_us": [ 26.49934375, 16.21615625, 27.67903125, 16.619140625, 25.205078125, 16.12109375, 26.212890625, 16.21028125, 25.0045625 ], "compute_throughput_us_p50": 25.0045625, "compute_throughput_us_p90": 26.73528125, "command_buffer_throughput_us": [ 26.50000078578125, 16.21679621171875, 27.679036975312503, 16.619140296875, 25.20572888890625, 16.1223952090625, 26.212891498281248, 16.21028604859375, 25.0052089540625 ], "command_buffer_throughput_us_p50": 25.0052089540625, "command_buffer_throughput_us_p90": 26.7358080236875, "compute_latency_us": [ 18.0, 153.458, 19.916, 20.042, 18.125, 17.916, 17.916, 20.0, 17.958 ], "compute_latency_us_p50": 18.125, "compute_latency_us_p90": 46.72520000000003, "command_buffer_latency_us": [ 18.0833158083, 153.541623149, 19.8750058189, 20.0416543521, 18.208345863999998, 17.9582857527, 17.9582857527, 20.0416543521, 17.958343960300002 ], "command_buffer_latency_us_p50": 18.208345863999998, "command_buffer_latency_us_p90": 46.74164811148002, "counter_control_throughput_ratio": 1.0075247983691418, "counter_control_latency_ratio": 0.7219415512857018 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 23.1671813685, "throughput_us_p90": 24.76098549052, "latency_us_p50": 254.459, "latency_us_p90": 282.9832 }, "torch": { "cold_call_ms": 6.175792, "warmup_ms": 200.391917, "repetitions": 471, "throughput_us": [ 66.00327388535031, 59.99256900212314, 64.075991507431, 60.902866242038215, 63.573602972399144, 60.46487898089172, 63.171798301486206, 60.248318471337576, 63.697452229299365 ], "latency_us": [ 485.416, 335.83299999999997, 332.208, 320.834, 309.333, 300.042, 299.70799999999997, 293.584, 314.916 ], "throughput_us_p50": 63.171798301486206, "throughput_us_p90": 64.46144798301486, "latency_us_p50": 314.916, "latency_us_p90": 365.7496, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2975749.9578408897, "command_buffers": 1 }, { "command_buffer_ns": 2691541.681997478, "command_buffers": 1 }, { "command_buffer_ns": 2731833.3741277456, "command_buffers": 1 }, { "command_buffer_ns": 3022791.7013689876, "command_buffers": 1 }, { "command_buffer_ns": 2917750.0400692225, "command_buffers": 1 }, { "command_buffer_ns": 2925208.304077387, "command_buffers": 1 }, { "command_buffer_ns": 2961416.670586914, "command_buffers": 1 }, { "command_buffer_ns": 2879916.6320823133, "command_buffers": 1 }, { "command_buffer_ns": 3027708.3278633654, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 202916.6789725423, "command_buffers": 1 }, { "command_buffer_ns": 251249.99228864908, "command_buffers": 1 }, { "command_buffer_ns": 44249.987695366144, "command_buffers": 1 }, { "command_buffer_ns": 43124.950025230646, "command_buffers": 1 }, { "command_buffer_ns": 237416.7088419199, "command_buffers": 1 }, { "command_buffer_ns": 48583.31521973014, "command_buffers": 1 }, { "command_buffer_ns": 43625.012040138245, "command_buffers": 1 }, { "command_buffer_ns": 42541.65105521679, "command_buffers": 1 }, { "command_buffer_ns": 44291.66438058019, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 46.4960930912639, 42.055338781210594, 42.684896470746025, 47.23112033389043, 45.5898443760816, 45.70637975120917, 46.27213547792053, 44.998697376286145, 47.307942622865085 ], "command_buffer_throughput_us_p50": 45.70637975120917, "command_buffer_throughput_us_p90": 47.24648479168536, "command_buffer_latency_us": [ 202.9166789725423, 251.24999228864908, 44.249987695366144, 43.124950025230646, 237.4167088419199, 48.58331521973014, 43.625012040138245, 42.54165105521679, 44.29166438058019 ], "command_buffer_latency_us_p50": 44.29166438058019, "command_buffer_latency_us_p90": 240.18336553126574 }, "throughput": [ { "compute_ns": 2876375.0, "compute_span_ns": 2876375.0, "command_buffer_ns": 2876375.0451616943, "calibration_cpu_ns": 4580459.0, "calibration_gpu_ticks": 4580459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 3148291.0, "compute_span_ns": 3148291.0, "command_buffer_ns": 3148249.990772456, "calibration_cpu_ns": 4578666.0, "calibration_gpu_ticks": 4578666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2923209.0, "compute_span_ns": 2923209.0, "command_buffer_ns": 2923250.023741275, "calibration_cpu_ns": 4080250.0, "calibration_gpu_ticks": 4080250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2966042.0, "compute_span_ns": 2966042.0, "command_buffer_ns": 2966041.676700115, "calibration_cpu_ns": 4608417.0, "calibration_gpu_ticks": 4608417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2978333.0, "compute_span_ns": 2978333.0, "command_buffer_ns": 2978291.653562337, "calibration_cpu_ns": 4112166.0, "calibration_gpu_ticks": 4112166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2904583.0, "compute_span_ns": 2904583.0, "command_buffer_ns": 2904583.3507552743, "calibration_cpu_ns": 4287125.0, "calibration_gpu_ticks": 4287125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2886291.0, "compute_span_ns": 2886291.0, "command_buffer_ns": 2886249.974835664, "calibration_cpu_ns": 4465208.0, "calibration_gpu_ticks": 4465208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2869375.0, "compute_span_ns": 2869375.0, "command_buffer_ns": 2869374.9918602407, "calibration_cpu_ns": 4232791.0, "calibration_gpu_ticks": 4232791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2787791.0, "compute_span_ns": 2787791.0, "command_buffer_ns": 2787749.981507659, "calibration_cpu_ns": 4005709.0, "calibration_gpu_ticks": 4005709.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 44916.0, "compute_span_ns": 44916.0, "command_buffer_ns": 44874.963350594044, "calibration_cpu_ns": 616708.0, "calibration_gpu_ticks": 616708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 42791.0, "compute_span_ns": 42791.0, "command_buffer_ns": 42749.97627362609, "calibration_cpu_ns": 609042.0, "calibration_gpu_ticks": 609042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 42875.0, "compute_span_ns": 42875.0, "command_buffer_ns": 42875.00632926822, "calibration_cpu_ns": 520875.0, "calibration_gpu_ticks": 520875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44625.0, "compute_span_ns": 44625.0, "command_buffer_ns": 44625.019654631615, "calibration_cpu_ns": 611917.0, "calibration_gpu_ticks": 611917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45083.0, "compute_span_ns": 45083.0, "command_buffer_ns": 45041.670091450214, "calibration_cpu_ns": 772625.0, "calibration_gpu_ticks": 772625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43417.0, "compute_span_ns": 43417.0, "command_buffer_ns": 43458.36350694299, "calibration_cpu_ns": 436000.0, "calibration_gpu_ticks": 436000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 208167.0, "compute_span_ns": 208167.0, "command_buffer_ns": 208166.66074097157, "calibration_cpu_ns": 1289584.0, "calibration_gpu_ticks": 1289584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44292.0, "compute_span_ns": 44292.0, "command_buffer_ns": 44333.34106579423, "calibration_cpu_ns": 614667.0, "calibration_gpu_ticks": 614667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45250.0, "compute_span_ns": 45250.0, "command_buffer_ns": 45249.995309859514, "calibration_cpu_ns": 535125.0, "calibration_gpu_ticks": 535125.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 44.943359375, 49.192046875, 45.675140625, 46.34440625, 46.536453125, 45.384109375, 45.098296875, 44.833984375, 43.559234375 ], "compute_throughput_us_p50": 45.384109375, "compute_throughput_us_p90": 47.067571875, "command_buffer_throughput_us": [ 44.94336008065147, 49.191406105819624, 45.67578162095742, 46.3444011984393, 46.53580708691152, 45.38411485555116, 45.09765585680725, 44.83398424781626, 43.55859346105717 ], "command_buffer_throughput_us_p50": 45.38411485555116, "command_buffer_throughput_us_p90": 47.06692689069314, "compute_latency_us": [ 44.916, 42.791, 42.875, 44.625, 45.083, 43.417, 208.167, 44.292, 45.25 ], "compute_latency_us_p50": 44.625, "compute_latency_us_p90": 77.83340000000003, "command_buffer_latency_us": [ 44.874963350594044, 42.74997627362609, 42.87500632926822, 44.625019654631615, 45.041670091450214, 43.45836350694299, 208.16666074097157, 44.33334106579423, 45.249995309859514 ], "command_buffer_latency_us_p50": 44.625019654631615, "command_buffer_latency_us_p90": 77.83332839608195, "counter_control_throughput_ratio": 0.9929492360276142, "counter_control_latency_ratio": 0.968926966136944 }, "download_ms": 0.359, "allocation_upload_ms": 1.32075, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.36673297248773995, "device_compute_slowdown": 0.5509541300765032, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.32225, "warmup_ms": 200.521458, "repetitions": 423, "throughput_us": [ 64.83884869976359, 61.62105910165485, 62.11327895981088, 61.51231205673759, 65.24891725768322, 58.731973995271865, 59.211879432624116, 62.00138061465721, 60.846827423167845 ], "latency_us": [ 476.791, 345.66700000000003, 295.66700000000003, 253.542, 245.417, 309.25, 275.54200000000003, 272.125, 257.041 ], "throughput_us_p50": 61.62105910165485, "throughput_us_p90": 64.92086241134751, "latency_us_p50": 275.54200000000003, "latency_us_p90": 371.89180000000005, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2903291.6645519435, "command_buffers": 1 }, { "command_buffer_ns": 2901625.004597008, "command_buffers": 1 }, { "command_buffer_ns": 2866458.3223871887, "command_buffers": 1 }, { "command_buffer_ns": 2959000.0049211085, "command_buffers": 1 }, { "command_buffer_ns": 2839250.024408102, "command_buffers": 1 }, { "command_buffer_ns": 2951624.9778680503, "command_buffers": 1 }, { "command_buffer_ns": 2957416.6983366013, "command_buffers": 1 }, { "command_buffer_ns": 5145166.70955345, "command_buffers": 1 }, { "command_buffer_ns": 4867166.688200086, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 42791.65295884013, "command_buffers": 1 }, { "command_buffer_ns": 43875.01394376159, "command_buffers": 1 }, { "command_buffer_ns": 45499.99721348286, "command_buffers": 1 }, { "command_buffer_ns": 48500.02005696297, "command_buffers": 1 }, { "command_buffer_ns": 44125.01584738493, "command_buffers": 1 }, { "command_buffer_ns": 48083.311412483454, "command_buffers": 1 }, { "command_buffer_ns": 43249.980080872774, "command_buffers": 1 }, { "command_buffer_ns": 216500.01872330904, "command_buffers": 1 }, { "command_buffer_ns": 44291.66438058019, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 45.36393225862412, 45.337890696828254, 44.78841128729982, 46.23437507689232, 44.363281631376594, 46.11914027918829, 46.209635911509395, 80.39322983677266, 76.04947950312635 ], "command_buffer_throughput_us_p50": 46.11914027918829, "command_buffer_throughput_us_p90": 76.91822956985561, "command_buffer_latency_us": [ 42.79165295884013, 43.87501394376159, 45.49999721348286, 48.50002005696297, 44.12501584738493, 48.083311412483454, 43.249980080872774, 216.50001872330904, 44.29166438058019 ], "command_buffer_latency_us_p50": 44.29166438058019, "command_buffer_latency_us_p90": 82.10001979023221 }, "throughput": [ { "compute_ns": 2989917.0, "compute_span_ns": 2989917.0, "command_buffer_ns": 2989916.654769331, "calibration_cpu_ns": 4455250.0, "calibration_gpu_ticks": 4455250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2957333.0, "compute_span_ns": 2957333.0, "command_buffer_ns": 2957291.668280959, "calibration_cpu_ns": 4288750.0, "calibration_gpu_ticks": 4288750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2964958.0, "compute_span_ns": 2964958.0, "command_buffer_ns": 2964916.6390299797, "calibration_cpu_ns": 4336125.0, "calibration_gpu_ticks": 4336125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2910583.0, "compute_span_ns": 2910583.0, "command_buffer_ns": 2910583.3382345736, "calibration_cpu_ns": 4558625.0, "calibration_gpu_ticks": 4558625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2889625.0, "compute_span_ns": 2889625.0, "command_buffer_ns": 2889625.0296384096, "calibration_cpu_ns": 4236584.0, "calibration_gpu_ticks": 4236584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2980250.0, "compute_span_ns": 2980250.0, "command_buffer_ns": 2980249.99210611, "calibration_cpu_ns": 4342709.0, "calibration_gpu_ticks": 4342709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2676625.0, "compute_span_ns": 2676625.0, "command_buffer_ns": 2676625.0375658274, "calibration_cpu_ns": 4825542.0, "calibration_gpu_ticks": 4825542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2854625.0, "compute_span_ns": 2854625.0, "command_buffer_ns": 2854624.9959617853, "calibration_cpu_ns": 4298667.0, "calibration_gpu_ticks": 4298667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5010750.0, "compute_span_ns": 5010750.0, "command_buffer_ns": 5010750.028304756, "calibration_cpu_ns": 7282083.0, "calibration_gpu_ticks": 7282083.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 43667.0, "compute_span_ns": 43667.0, "command_buffer_ns": 43708.36541056633, "calibration_cpu_ns": 518167.0, "calibration_gpu_ticks": 518167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50792.0, "compute_span_ns": 50792.0, "command_buffer_ns": 50791.65566712618, "calibration_cpu_ns": 3355667.0, "calibration_gpu_ticks": 3355667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47625.0, "compute_span_ns": 47625.0, "command_buffer_ns": 47624.98429045081, "calibration_cpu_ns": 475500.0, "calibration_gpu_ticks": 475500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 46417.0, "compute_span_ns": 46417.0, "command_buffer_ns": 46458.328142762184, "calibration_cpu_ns": 563833.0, "calibration_gpu_ticks": 563833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43875.0, "compute_span_ns": 43875.0, "command_buffer_ns": 43875.01394376159, "calibration_cpu_ns": 567125.0, "calibration_gpu_ticks": 567125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 290709.0, "compute_span_ns": 290709.0, "command_buffer_ns": 290750.00202283263, "calibration_cpu_ns": 1599875.0, "calibration_gpu_ticks": 1599875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44583.0, "compute_span_ns": 44583.0, "command_buffer_ns": 44541.66628420353, "calibration_cpu_ns": 471083.0, "calibration_gpu_ticks": 471083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43166.0, "compute_span_ns": 43166.0, "command_buffer_ns": 43125.00823289156, "calibration_cpu_ns": 580250.0, "calibration_gpu_ticks": 580250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43125.0, "compute_span_ns": 43125.0, "command_buffer_ns": 43125.00823289156, "calibration_cpu_ns": 507167.0, "calibration_gpu_ticks": 507167.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 46.717453125, 46.208328125, 46.32746875, 45.477859375, 45.150390625, 46.56640625, 41.822265625, 44.603515625, 78.29296875 ], "compute_throughput_us_p50": 46.208328125, "compute_throughput_us_p90": 53.032556250000006, "command_buffer_throughput_us": [ 46.7174477307708, 46.207682316889986, 46.32682248484343, 45.47786465991521, 45.15039108810015, 46.56640612665797, 41.82226621196605, 44.603515561902896, 78.29296919226181 ], "command_buffer_throughput_us_p50": 46.207682316889986, "command_buffer_throughput_us_p90": 53.03255202306901, "compute_latency_us": [ 43.667, 50.792, 47.625, 46.417, 43.875, 290.709, 44.583, 43.166, 43.125 ], "compute_latency_us_p50": 44.583, "compute_latency_us_p90": 98.77540000000005, "command_buffer_latency_us": [ 43.70836541056633, 50.79165566712618, 47.62498429045081, 46.458328142762184, 43.87501394376159, 290.75000202283263, 44.54166628420353, 43.12500823289156, 43.12500823289156 ], "command_buffer_latency_us_p50": 44.54166628420353, "command_buffer_latency_us_p90": 98.78332493826751, "counter_control_throughput_ratio": 1.0177423632287577, "counter_control_latency_ratio": 1.0214226931735484 }, "download_ms": 0.377958, "allocation_upload_ms": 0.803583, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-avggotcn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b8d9d984f3aaf8bfdb9269f6e0efc630605362c6ff4e852cc112881dfaf5e97b", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.270916, "capture_ms": 0.078542, "compile_ms": 65.027542, "allocation_upload_ms": 3.869416, "cold_call_ms": 8.562291, "warmup_ms": 200.362292, "download_ms": 0.784458, "repetitions": 1484, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 24.9191374663, 22.8936152291, 23.0980175202, 22.8821314016, 23.0559858491, 23.9276165768, 22.5318113208, 23.3479332884, 22.8765161725 ], "latency_us": [ 445.875, 319, 253.709, 245.542, 245.625, 232, 245.125, 235.083, 207.292 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1405667, "compute_span_ns": 1405667, "calibration_cpu_ns": 2687792, "calibration_gpu_ticks": 2687792, "compute_passes": 1, "command_buffer_ns": 1405708.31718, "command_buffers": 2 }, { "compute_ns": 1067375, "compute_span_ns": 1067375, "calibration_cpu_ns": 1640875, "calibration_gpu_ticks": 1640875, "compute_passes": 1, "command_buffer_ns": 1067416.68424, "command_buffers": 2 }, { "compute_ns": 1688500, "compute_span_ns": 1688500, "calibration_cpu_ns": 3008084, "calibration_gpu_ticks": 3008084, "compute_passes": 1, "command_buffer_ns": 1688624.96503, "command_buffers": 2 }, { "compute_ns": 1024083, "compute_span_ns": 1024083, "calibration_cpu_ns": 1642292, "calibration_gpu_ticks": 1642292, "compute_passes": 1, "command_buffer_ns": 1024083.35079, "command_buffers": 2 }, { "compute_ns": 1672833, "compute_span_ns": 1672833, "calibration_cpu_ns": 2651292, "calibration_gpu_ticks": 2651292, "compute_passes": 1, "command_buffer_ns": 1672874.96151, "command_buffers": 2 }, { "compute_ns": 1052084, "compute_span_ns": 1052084, "calibration_cpu_ns": 1487875, "calibration_gpu_ticks": 1487875, "compute_passes": 1, "command_buffer_ns": 1052166.68453, "command_buffers": 2 }, { "compute_ns": 1606958, "compute_span_ns": 1606958, "calibration_cpu_ns": 2817292, "calibration_gpu_ticks": 2817292, "compute_passes": 1, "command_buffer_ns": 1607041.63136, "command_buffers": 2 }, { "compute_ns": 1046750, "compute_span_ns": 1046750, "calibration_cpu_ns": 1518167, "calibration_gpu_ticks": 1518167, "compute_passes": 1, "command_buffer_ns": 1046875.02608, "command_buffers": 2 }, { "compute_ns": 1653625, "compute_span_ns": 1653625, "calibration_cpu_ns": 2945375, "calibration_gpu_ticks": 2945375, "compute_passes": 1, "command_buffer_ns": 1653708.34293, "command_buffers": 2 } ], "latency": [ { "compute_ns": 17625, "compute_span_ns": 17625, "calibration_cpu_ns": 452459, "calibration_gpu_ticks": 452459, "compute_passes": 1, "command_buffer_ns": 17666.6653715, "command_buffers": 2 }, { "compute_ns": 272666, "compute_span_ns": 272666, "calibration_cpu_ns": 1597750, "calibration_gpu_ticks": 1597750, "compute_passes": 1, "command_buffer_ns": 272708.3629, "command_buffers": 2 }, { "compute_ns": 20042, "compute_span_ns": 20042, "calibration_cpu_ns": 634125, "calibration_gpu_ticks": 634125, "compute_passes": 1, "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "compute_ns": 20333, "compute_span_ns": 20333, "calibration_cpu_ns": 418458, "calibration_gpu_ticks": 418458, "compute_passes": 1, "command_buffer_ns": 20416.6281037, "command_buffers": 2 }, { "compute_ns": 21875, "compute_span_ns": 21875, "calibration_cpu_ns": 422375, "calibration_gpu_ticks": 422375, "compute_passes": 1, "command_buffer_ns": 21916.6977331, "command_buffers": 2 }, { "compute_ns": 17708, "compute_span_ns": 17708, "calibration_cpu_ns": 533542, "calibration_gpu_ticks": 533542, "compute_passes": 1, "command_buffer_ns": 17791.6372195, "command_buffers": 2 }, { "compute_ns": 188875, "compute_span_ns": 188875, "calibration_cpu_ns": 1613666, "calibration_gpu_ticks": 1613666, "compute_passes": 1, "command_buffer_ns": 188999.983948, "command_buffers": 2 }, { "compute_ns": 22708, "compute_span_ns": 22708, "calibration_cpu_ns": 663583, "calibration_gpu_ticks": 663583, "compute_passes": 1, "command_buffer_ns": 22916.6471399, "command_buffers": 2 }, { "compute_ns": 20125, "compute_span_ns": 20125, "calibration_cpu_ns": 425250, "calibration_gpu_ticks": 425250, "compute_passes": 1, "command_buffer_ns": 20166.6844077, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1277708.33205, "command_buffers": 2 }, { "command_buffer_ns": 1655458.35625, "command_buffers": 2 }, { "command_buffer_ns": 1019041.69424, "command_buffers": 2 }, { "command_buffer_ns": 1689791.67938, "command_buffers": 2 }, { "command_buffer_ns": 1041541.69094, "command_buffers": 2 }, { "command_buffer_ns": 1596000.04554, "command_buffers": 2 }, { "command_buffer_ns": 1042750.02377, "command_buffers": 2 }, { "command_buffer_ns": 1578124.99674, "command_buffers": 2 }, { "command_buffer_ns": 1033041.68442, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 18541.701138, "command_buffers": 2 }, { "command_buffer_ns": 22958.2656175, "command_buffers": 2 }, { "command_buffer_ns": 17750.0187419, "command_buffers": 2 }, { "command_buffer_ns": 27125.0028163, "command_buffers": 2 }, { "command_buffer_ns": 40374.987293, "command_buffers": 2 }, { "command_buffer_ns": 35291.7122655, "command_buffers": 2 }, { "command_buffer_ns": 20124.9495149, "command_buffers": 2 }, { "command_buffer_ns": 17875.0487976, "command_buffers": 2 }, { "command_buffer_ns": 20208.361093, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 19.96419268828125, 25.86653681640625, 15.9225264725, 26.4029949903125, 16.2740889209375, 24.9375007115625, 16.292969121406248, 24.658203074062502, 16.1412763190625 ], "command_buffer_throughput_us_p50": 19.96419268828125, "command_buffer_throughput_us_p90": 25.9738284511875, "command_buffer_latency_us": [ 18.541701138, 22.9582656175, 17.7500187419, 27.1250028163, 40.374987293, 35.291712265499996, 20.1249495149, 17.8750487976, 20.208361093 ], "command_buffer_latency_us_p50": 20.208361093, "command_buffer_latency_us_p90": 36.308367270999995 }, "compute_throughput_us": [ 21.963546875, 16.677734375, 26.3828125, 16.001296875, 26.138015625, 16.4388125, 25.10871875, 16.35546875, 25.837890625 ], "compute_throughput_us_p50": 21.963546875, "compute_throughput_us_p90": 26.186975, "command_buffer_throughput_us": [ 21.9641924559375, 16.67838569125, 26.38476507859375, 16.001302356093753, 26.13867127359375, 16.440104445781248, 25.11002549, 16.3574222825, 25.83919285828125 ], "command_buffer_throughput_us_p50": 21.9641924559375, "command_buffer_throughput_us_p90": 26.18789003459375, "compute_latency_us": [ 17.625, 272.666, 20.042, 20.333, 21.875, 17.708, 188.875, 22.708, 20.125 ], "compute_latency_us_p50": 20.333, "compute_latency_us_p90": 205.63320000000002, "command_buffer_latency_us": [ 17.6666653715, 272.7083629, 20.125007722499998, 20.4166281037, 21.9166977331, 17.7916372195, 188.99998394800002, 22.9166471399, 20.1666844077 ], "command_buffer_latency_us_p50": 20.4166281037, "command_buffer_latency_us_p90": 205.74165973840002, "counter_control_throughput_ratio": 1.1001793460363778, "counter_control_latency_ratio": 0.9979376513954693 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 23.0559858491, "throughput_us_p90": 24.125920754699997, "latency_us_p50": 245.542, "latency_us_p90": 344.375 }, "slowdown": 0.3741575718629741, "device_compute_slowdown": 0.4753157659282874, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-au_qh5zh/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "2f415c8199d32d1b999ce6dae6674c9bc8bbdf9b0adda641a6d2aee649468028", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.784584, "capture_ms": 0.078042, "compile_ms": 64.189917, "allocation_upload_ms": 8.408459, "cold_call_ms": 8.430583, "warmup_ms": 201.84075, "download_ms": 3.603625, "repetitions": 284, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 109.367221831, 110.10578169, 100.959651408, 110.136295775, 109.243542254, 101.040785211, 110.224911972, 104.149207746, 105.681193662 ], "latency_us": [ 372.959, 310.167, 312.5, 297.708, 2091.125, 949.5, 498.875, 393.875, 315.042 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 5286625, "compute_span_ns": 5286625, "calibration_cpu_ns": 6430250, "calibration_gpu_ticks": 6430250, "compute_passes": 1, "command_buffer_ns": 5286666.68106, "command_buffers": 2 }, { "compute_ns": 5261000, "compute_span_ns": 5261000, "calibration_cpu_ns": 6122500, "calibration_gpu_ticks": 6122500, "compute_passes": 1, "command_buffer_ns": 5261083.30814, "command_buffers": 2 }, { "compute_ns": 5219583, "compute_span_ns": 5219583, "calibration_cpu_ns": 6160916, "calibration_gpu_ticks": 6160916, "compute_passes": 1, "command_buffer_ns": 5219541.6647, "command_buffers": 2 }, { "compute_ns": 5361708, "compute_span_ns": 5361708, "calibration_cpu_ns": 6429417, "calibration_gpu_ticks": 6429417, "compute_passes": 1, "command_buffer_ns": 5361791.58371, "command_buffers": 2 }, { "compute_ns": 5234625, "compute_span_ns": 5234625, "calibration_cpu_ns": 6337833, "calibration_gpu_ticks": 6337833, "compute_passes": 1, "command_buffer_ns": 5234625.01587, "command_buffers": 2 }, { "compute_ns": 5242041, "compute_span_ns": 5242041, "calibration_cpu_ns": 6285875, "calibration_gpu_ticks": 6285875, "compute_passes": 1, "command_buffer_ns": 5242041.6614, "command_buffers": 2 }, { "compute_ns": 5295000, "compute_span_ns": 5295000, "calibration_cpu_ns": 6182417, "calibration_gpu_ticks": 6182417, "compute_passes": 1, "command_buffer_ns": 5295041.65752, "command_buffers": 2 }, { "compute_ns": 5333584, "compute_span_ns": 5333584, "calibration_cpu_ns": 9801416, "calibration_gpu_ticks": 9801416, "compute_passes": 1, "command_buffer_ns": 5333624.9548, "command_buffers": 2 }, { "compute_ns": 7268291, "compute_span_ns": 7268291, "calibration_cpu_ns": 8441750, "calibration_gpu_ticks": 8441750, "compute_passes": 1, "command_buffer_ns": 7268291.66524, "command_buffers": 2 } ], "latency": [ { "compute_ns": 116416, "compute_span_ns": 116416, "calibration_cpu_ns": 1080583, "calibration_gpu_ticks": 1080583, "compute_passes": 1, "command_buffer_ns": 116499.955766, "command_buffers": 2 }, { "compute_ns": 505333, "compute_span_ns": 505333, "calibration_cpu_ns": 1825083, "calibration_gpu_ticks": 1825083, "compute_passes": 1, "command_buffer_ns": 505416.65405, "command_buffers": 2 }, { "compute_ns": 73333, "compute_span_ns": 73333, "calibration_cpu_ns": 686042, "calibration_gpu_ticks": 686042, "compute_passes": 1, "command_buffer_ns": 73333.3290555, "command_buffers": 2 }, { "compute_ns": 112750, "compute_span_ns": 112750, "calibration_cpu_ns": 1050041, "calibration_gpu_ticks": 1050041, "compute_passes": 1, "command_buffer_ns": 112791.720312, "command_buffers": 2 }, { "compute_ns": 72833, "compute_span_ns": 72833, "calibration_cpu_ns": 640041, "calibration_gpu_ticks": 640041, "compute_passes": 1, "command_buffer_ns": 72916.6786186, "command_buffers": 2 }, { "compute_ns": 115083, "compute_span_ns": 115083, "calibration_cpu_ns": 1022375, "calibration_gpu_ticks": 1022375, "compute_passes": 1, "command_buffer_ns": 115125.032607, "command_buffers": 2 }, { "compute_ns": 114334, "compute_span_ns": 114334, "calibration_cpu_ns": 3102375, "calibration_gpu_ticks": 3102375, "compute_passes": 1, "command_buffer_ns": 114416.645374, "command_buffers": 2 }, { "compute_ns": 74666, "compute_span_ns": 74666, "calibration_cpu_ns": 515792, "calibration_gpu_ticks": 515792, "compute_passes": 1, "command_buffer_ns": 74625.0152588, "command_buffers": 2 }, { "compute_ns": 72916, "compute_span_ns": 72916, "calibration_cpu_ns": 499292, "calibration_gpu_ticks": 499292, "compute_passes": 1, "command_buffer_ns": 72916.6786186, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5338125.04727, "command_buffers": 2 }, { "command_buffer_ns": 5325666.68699, "command_buffers": 2 }, { "command_buffer_ns": 5428333.3593, "command_buffers": 2 }, { "command_buffer_ns": 5478083.33067, "command_buffers": 2 }, { "command_buffer_ns": 5455791.65919, "command_buffers": 2 }, { "command_buffer_ns": 5314291.68768, "command_buffers": 2 }, { "command_buffer_ns": 5419000.05192, "command_buffers": 2 }, { "command_buffer_ns": 7437625.0268, "command_buffers": 2 }, { "command_buffer_ns": 7347416.71477, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 117541.640066, "command_buffers": 2 }, { "command_buffer_ns": 115458.329674, "command_buffers": 2 }, { "command_buffer_ns": 74541.6618884, "command_buffers": 2 }, { "command_buffer_ns": 75416.6394472, "command_buffers": 2 }, { "command_buffer_ns": 114916.649181, "command_buffers": 2 }, { "command_buffer_ns": 116625.044029, "command_buffers": 2 }, { "command_buffer_ns": 74958.3705328, "command_buffers": 2 }, { "command_buffer_ns": 72041.6428521, "command_buffers": 2 }, { "command_buffer_ns": 115208.32777, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 83.40820386359376, 83.21354198421875, 84.8177087390625, 85.59505204171874, 85.24674467484375, 83.03580762, 84.67187581125, 116.21289104375, 114.80338616828125 ], "command_buffer_throughput_us_p50": 84.8177087390625, "command_buffer_throughput_us_p90": 115.085287143375, "command_buffer_latency_us": [ 117.54164006600001, 115.458329674, 74.5416618884, 75.4166394472, 114.916649181, 116.625044029, 74.9583705328, 72.04164285210001, 115.20832777 ], "command_buffer_latency_us_p50": 114.916649181, "command_buffer_latency_us_p90": 116.8083632364 }, "compute_throughput_us": [ 82.603515625, 82.203125, 81.555984375, 83.7766875, 81.791015625, 81.906890625, 82.734375, 83.33725, 113.567046875 ], "compute_throughput_us_p50": 82.603515625, "compute_throughput_us_p90": 89.734759375, "command_buffer_throughput_us": [ 82.6041668915625, 82.20442668968751, 81.55533851093749, 83.77799349546875, 81.79101587296876, 81.906900959375, 82.73502589875, 83.33788991875001, 113.567057269375 ], "command_buffer_throughput_us_p50": 82.6041668915625, "command_buffer_throughput_us_p90": 89.73580625025001, "compute_latency_us": [ 116.416, 505.333, 73.333, 112.75, 72.833, 115.083, 114.334, 74.666, 72.916 ], "compute_latency_us_p50": 112.75, "compute_latency_us_p90": 194.19940000000008, "command_buffer_latency_us": [ 116.499955766, 505.41665405000003, 73.3333290555, 112.79172031200001, 72.9166786186, 115.125032607, 114.416645374, 74.6250152588, 72.9166786186 ], "command_buffer_latency_us_p50": 112.79172031200001, "command_buffer_latency_us_p90": 194.28329542280008, "counter_control_throughput_ratio": 0.9787714534554594, "counter_control_latency_ratio": 0.9911377423403732 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 109.243542254, "throughput_us_p90": 110.15401901439999, "latency_us_p50": 372.959, "latency_us_p90": 1177.8250000000003 }, "torch": { "cold_call_ms": 7.540958, "warmup_ms": 201.348125, "repetitions": 78, "throughput_us": [ 397.95780769230765, 383.35096153846155, 380.3296025641026, 402.6768205128205, 368.73878205128204, 403.8039615384615, 397.4097307692308, 367.3947692307692, 402.36110256410257 ], "latency_us": [ 612.9590000000001, 509.20799999999997, 509.20799999999997, 510.459, 2751.7079999999996, 718.7909999999999, 599.292, 499.29200000000003, 581.542 ], "throughput_us_p50": 397.4097307692308, "throughput_us_p90": 402.9022487179487, "latency_us_p50": 581.542, "latency_us_p90": 1125.3744000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 21976458.374410868, "command_buffers": 1 }, { "command_buffer_ns": 22432000.027038157, "command_buffers": 1 }, { "command_buffer_ns": 24229749.978985637, "command_buffers": 1 }, { "command_buffer_ns": 24209624.97126311, "command_buffers": 1 }, { "command_buffer_ns": 22178583.312779665, "command_buffers": 1 }, { "command_buffer_ns": 24405375.006608665, "command_buffers": 1 }, { "command_buffer_ns": 21802000.00293553, "command_buffers": 1 }, { "command_buffer_ns": 24177750.04869327, "command_buffers": 1 }, { "command_buffer_ns": 23202416.66166112, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 272041.6523516178, "command_buffers": 1 }, { "command_buffer_ns": 286041.7007468641, "command_buffers": 1 }, { "command_buffer_ns": 279125.0008158386, "command_buffers": 1 }, { "command_buffer_ns": 271875.00381842256, "command_buffers": 1 }, { "command_buffer_ns": 271375.00001117587, "command_buffers": 1 }, { "command_buffer_ns": 282499.99741092324, "command_buffers": 1 }, { "command_buffer_ns": 271250.02816319466, "command_buffers": 1 }, { "command_buffer_ns": 683041.6969023645, "command_buffers": 1 }, { "command_buffer_ns": 275875.03427639604, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 343.3821621001698, 350.5000004224712, 378.5898434216506, 378.2753901759861, 346.54036426218227, 381.3339844782604, 340.65625004586764, 377.77734451083234, 362.537760338455 ], "command_buffer_throughput_us_p50": 362.537760338455, "command_buffer_throughput_us_p90": 379.13867163297255, "command_buffer_latency_us": [ 272.0416523516178, 286.0417007468641, 279.1250008158386, 271.87500381842256, 271.37500001117587, 282.49999741092324, 271.25002816319466, 683.0416969023645, 275.87503427639604 ], "command_buffer_latency_us_p50": 275.87503427639604, "command_buffer_latency_us_p90": 365.4416999779642 }, "throughput": [ { "compute_ns": 22117958.0, "compute_span_ns": 22117958.0, "command_buffer_ns": 22117916.66923091, "calibration_cpu_ns": 23761000.0, "calibration_gpu_ticks": 23761000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24297375.0, "compute_span_ns": 24297375.0, "command_buffer_ns": 24297374.999150634, "calibration_cpu_ns": 25613917.0, "calibration_gpu_ticks": 25613917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24386750.0, "compute_span_ns": 24386750.0, "command_buffer_ns": 24386750.01030788, "calibration_cpu_ns": 26445250.0, "calibration_gpu_ticks": 26445250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24504708.0, "compute_span_ns": 24504708.0, "command_buffer_ns": 24504708.35901797, "calibration_cpu_ns": 25599167.0, "calibration_gpu_ticks": 25599167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24283417.0, "compute_span_ns": 24283417.0, "command_buffer_ns": 24283416.685648263, "calibration_cpu_ns": 25900125.0, "calibration_gpu_ticks": 25900125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24356458.0, "compute_span_ns": 24356458.0, "command_buffer_ns": 24356416.65942967, "calibration_cpu_ns": 26624167.0, "calibration_gpu_ticks": 26624167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21767583.0, "compute_span_ns": 21767583.0, "command_buffer_ns": 21767541.649751365, "calibration_cpu_ns": 22834584.0, "calibration_gpu_ticks": 22834584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22122916.0, "compute_span_ns": 22122916.0, "command_buffer_ns": 22122874.9724105, "calibration_cpu_ns": 24104333.0, "calibration_gpu_ticks": 24104333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21995709.0, "compute_span_ns": 21995709.0, "command_buffer_ns": 21995749.964844435, "calibration_cpu_ns": 24189583.0, "calibration_gpu_ticks": 24189583.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 270375.0, "compute_span_ns": 270375.0, "command_buffer_ns": 270375.0506043434, "calibration_cpu_ns": 702458.0, "calibration_gpu_ticks": 702458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 284167.0, "compute_span_ns": 284167.0, "command_buffer_ns": 284166.65736585855, "calibration_cpu_ns": 1912958.0, "calibration_gpu_ticks": 1912958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 274250.0, "compute_span_ns": 274250.0, "command_buffer_ns": 274249.99279901385, "calibration_cpu_ns": 835209.0, "calibration_gpu_ticks": 835209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 282125.0, "compute_span_ns": 282125.0, "command_buffer_ns": 282125.0236593187, "calibration_cpu_ns": 1185709.0, "calibration_gpu_ticks": 1185709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 287375.0, "compute_span_ns": 287375.0, "command_buffer_ns": 287375.00542774796, "calibration_cpu_ns": 966958.0, "calibration_gpu_ticks": 966958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271375.0, "compute_span_ns": 271375.0, "command_buffer_ns": 271375.00001117587, "calibration_cpu_ns": 730125.0, "calibration_gpu_ticks": 730125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 284167.0, "compute_span_ns": 284167.0, "command_buffer_ns": 284208.3340510726, "calibration_cpu_ns": 1041000.0, "calibration_gpu_ticks": 1041000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270958.0, "compute_span_ns": 270958.0, "command_buffer_ns": 270916.6728891432, "calibration_cpu_ns": 717084.0, "calibration_gpu_ticks": 717084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271666.0, "compute_span_ns": 271666.0, "command_buffer_ns": 271625.0019147992, "calibration_cpu_ns": 738333.0, "calibration_gpu_ticks": 738333.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 345.59309375, 379.646484375, 381.04296875, 382.8860625, 379.428390625, 380.56965625, 340.118484375, 345.6705625, 343.682953125 ], "compute_throughput_us_p50": 379.428390625, "compute_throughput_us_p90": 381.4115875, "command_buffer_throughput_us": [ 345.59244795673294, 379.64648436172865, 381.0429689110606, 382.88606810965575, 379.4283857132541, 380.5690103035886, 340.1178382773651, 345.66992144391406, 343.6835932006943 ], "command_buffer_throughput_us_p50": 379.4283857132541, "command_buffer_throughput_us_p90": 381.41158875077963, "compute_latency_us": [ 270.375, 284.167, 274.25, 282.125, 287.375, 271.375, 284.167, 270.958, 271.666 ], "compute_latency_us_p50": 274.25, "compute_latency_us_p90": 284.80859999999996, "command_buffer_latency_us": [ 270.3750506043434, 284.16665736585855, 274.24999279901385, 282.1250236593187, 287.37500542774796, 271.37500001117587, 284.2083340510726, 270.91667288914323, 271.6250019147992 ], "command_buffer_latency_us_p50": 274.24999279901385, "command_buffer_latency_us_p90": 284.84166832640767, "counter_control_throughput_ratio": 1.006436810354518, "counter_control_latency_ratio": 0.993444860046246 }, "download_ms": 0.707875, "allocation_upload_ms": 17.865875, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.27488894658554774, "device_compute_slowdown": 0.21770515245033265, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.773166, "warmup_ms": 201.176834, "repetitions": 73, "throughput_us": [ 374.0285342465753, 409.96690410958905, 375.246, 392.5428082191781, 380.96690410958905, 373.1095890410959, 413.49087671232877, 376.63298630136984, 389.5473698630137 ], "latency_us": [ 1454.625, 596.4580000000001, 704.125, 552.333, 544.792, 511.375, 500.54200000000003, 495.70799999999997, 496.875 ], "throughput_us_p50": 380.96690410958905, "throughput_us_p90": 410.671698630137, "latency_us_p50": 544.792, "latency_us_p90": 854.2250000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 22280125.005636364, "command_buffers": 1 }, { "command_buffer_ns": 22147374.984342605, "command_buffers": 1 }, { "command_buffer_ns": 22165958.303958178, "command_buffers": 1 }, { "command_buffer_ns": 24316374.969203025, "command_buffers": 1 }, { "command_buffer_ns": 24352291.715331376, "command_buffers": 1 }, { "command_buffer_ns": 22080624.99994412, "command_buffers": 1 }, { "command_buffer_ns": 21863250.00366196, "command_buffers": 1 }, { "command_buffer_ns": 24247791.61810875, "command_buffers": 1 }, { "command_buffer_ns": 23893625.009804964, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 277499.9593384564, "command_buffers": 1 }, { "command_buffer_ns": 270874.9962039292, "command_buffers": 1 }, { "command_buffer_ns": 605958.339292556, "command_buffers": 1 }, { "command_buffer_ns": 270999.9680519104, "command_buffers": 1 }, { "command_buffer_ns": 738791.6557490826, "command_buffers": 1 }, { "command_buffer_ns": 672374.9684169888, "command_buffers": 1 }, { "command_buffer_ns": 278749.9688565731, "command_buffers": 1 }, { "command_buffer_ns": 274166.63942858577, "command_buffers": 1 }, { "command_buffer_ns": 270833.31951871514, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 348.1269532130682, 346.0527341303532, 346.3430984993465, 379.94335889379727, 380.50455805205274, 345.0097656241269, 341.61328130721813, 378.8717440329492, 373.33789077820256 ], "command_buffer_throughput_us_p50": 348.1269532130682, "command_buffer_throughput_us_p90": 380.05559872544836, "command_buffer_latency_us": [ 277.4999593384564, 270.8749962039292, 605.958339292556, 270.9999680519104, 738.7916557490826, 672.3749684169888, 278.7499688565731, 274.16663942858577, 270.83331951871514 ], "command_buffer_latency_us_p50": 277.4999593384564, "command_buffer_latency_us_p90": 685.6583058834076 }, "throughput": [ { "compute_ns": 22071459.0, "compute_span_ns": 22071459.0, "command_buffer_ns": 22071500.01777336, "calibration_cpu_ns": 23560833.0, "calibration_gpu_ticks": 23560833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21984917.0, "compute_span_ns": 21984917.0, "command_buffer_ns": 21984916.646033525, "calibration_cpu_ns": 23849708.0, "calibration_gpu_ticks": 23849708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22475333.0, "compute_span_ns": 22475333.0, "command_buffer_ns": 22475333.360489458, "calibration_cpu_ns": 25543209.0, "calibration_gpu_ticks": 25543209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24280500.0, "compute_span_ns": 24280500.0, "command_buffer_ns": 24280500.01617521, "calibration_cpu_ns": 26630625.0, "calibration_gpu_ticks": 26630625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24278292.0, "compute_span_ns": 24278292.0, "command_buffer_ns": 24278333.35241303, "calibration_cpu_ns": 28388792.0, "calibration_gpu_ticks": 28388792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24236041.0, "compute_span_ns": 24236041.0, "command_buffer_ns": 24235999.96836856, "calibration_cpu_ns": 25308792.0, "calibration_gpu_ticks": 25308792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 22075209.0, "compute_span_ns": 22075209.0, "command_buffer_ns": 22075249.98812005, "calibration_cpu_ns": 24798125.0, "calibration_gpu_ticks": 24798125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20084959.0, "compute_span_ns": 20084959.0, "command_buffer_ns": 20084999.967366457, "calibration_cpu_ns": 24915083.0, "calibration_gpu_ticks": 24915083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 24124208.0, "compute_span_ns": 24124208.0, "command_buffer_ns": 24124166.63719341, "calibration_cpu_ns": 25867792.0, "calibration_gpu_ticks": 25867792.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 643750.0, "compute_span_ns": 643750.0, "command_buffer_ns": 643750.0123865902, "calibration_cpu_ns": 1828042.0, "calibration_gpu_ticks": 1828042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 276208.0, "compute_span_ns": 276208.0, "command_buffer_ns": 276166.6546575725, "calibration_cpu_ns": 751459.0, "calibration_gpu_ticks": 751459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 276500.0, "compute_span_ns": 276500.0, "command_buffer_ns": 276500.00993162394, "calibration_cpu_ns": 705500.0, "calibration_gpu_ticks": 705500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 272458.0, "compute_span_ns": 272458.0, "command_buffer_ns": 272416.62610322237, "calibration_cpu_ns": 736458.0, "calibration_gpu_ticks": 736458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275958.0, "compute_span_ns": 275958.0, "command_buffer_ns": 275916.7109616101, "calibration_cpu_ns": 885041.0, "calibration_gpu_ticks": 885041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271042.0, "compute_span_ns": 271042.0, "command_buffer_ns": 271041.70294478536, "calibration_cpu_ns": 789792.0, "calibration_gpu_ticks": 789792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 277500.0, "compute_span_ns": 277500.0, "command_buffer_ns": 277500.0175461173, "calibration_cpu_ns": 1068333.0, "calibration_gpu_ticks": 1068333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 577792.0, "compute_span_ns": 577792.0, "command_buffer_ns": 577833.3288617432, "calibration_cpu_ns": 2189625.0, "calibration_gpu_ticks": 2189625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 952416.0, "compute_span_ns": 952416.0, "command_buffer_ns": 952375.0049993396, "calibration_cpu_ns": 2216500.0, "calibration_gpu_ticks": 2216500.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 344.866546875, 343.514328125, 351.177078125, 379.3828125, 379.3483125, 378.688140625, 344.925140625, 313.827484375, 376.94075 ], "compute_throughput_us_p50": 351.177078125, "compute_throughput_us_p90": 379.3552125, "command_buffer_throughput_us": [ 344.86718777770875, 343.51432259427384, 351.1770837576478, 379.38281275273766, 379.34895863145357, 378.68749950575875, 344.92578106437577, 313.8281244901009, 376.94010370614706 ], "command_buffer_throughput_us_p50": 351.1770837576478, "command_buffer_throughput_us_p90": 379.3557294557104, "compute_latency_us": [ 643.75, 276.208, 276.5, 272.458, 275.958, 271.042, 277.5, 577.792, 952.416 ], "compute_latency_us_p50": 276.5, "compute_latency_us_p90": 705.4832000000001, "command_buffer_latency_us": [ 643.7500123865902, 276.1666546575725, 276.50000993162394, 272.41662610322237, 275.9167109616101, 271.04170294478536, 277.5000175461173, 577.8333288617432, 952.3750049993396 ], "command_buffer_latency_us_p50": 276.50000993162394, "command_buffer_latency_us_p90": 705.4750109091401, "counter_control_throughput_ratio": 0.9985246586683562, "counter_control_latency_ratio": 1.0052275211008166 }, "download_ms": 0.968125, "allocation_upload_ms": 2.186333, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-rzpa3liq/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "4f0c4e70dcb06b507aa82e242c8819acd3628d5f79a065eb6c2f6b73ec6f396c", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.97, "capture_ms": 0.071958, "compile_ms": 63.756416, "allocation_upload_ms": 6.57175, "cold_call_ms": 4.841958, "warmup_ms": 203.134375, "download_ms": 3.253, "repetitions": 284, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 107.049880282, 110.977257042, 111.626908451, 102.221098592, 112.183242958, 112.481514085, 113.186327465, 103.624855634, 94.518193662 ], "latency_us": [ 341.917, 414.709, 354.583, 344.791, 354.667, 326.083, 319.584, 333.458, 2937.709 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 7298708, "compute_span_ns": 7298708, "calibration_cpu_ns": 8075292, "calibration_gpu_ticks": 8075292, "compute_passes": 1, "command_buffer_ns": 7298708.36949, "command_buffers": 2 }, { "compute_ns": 7313375, "compute_span_ns": 7313375, "calibration_cpu_ns": 8049792, "calibration_gpu_ticks": 8049792, "compute_passes": 1, "command_buffer_ns": 7313375.01202, "command_buffers": 2 }, { "compute_ns": 7271833, "compute_span_ns": 7271833, "calibration_cpu_ns": 8016500, "calibration_gpu_ticks": 8016500, "compute_passes": 1, "command_buffer_ns": 7271874.98705, "command_buffers": 2 }, { "compute_ns": 7323708, "compute_span_ns": 7323708, "calibration_cpu_ns": 8153166, "calibration_gpu_ticks": 8153166, "compute_passes": 1, "command_buffer_ns": 7323708.38523, "command_buffers": 2 }, { "compute_ns": 7281292, "compute_span_ns": 7281292, "calibration_cpu_ns": 8040459, "calibration_gpu_ticks": 8040459, "compute_passes": 1, "command_buffer_ns": 7281416.67787, "command_buffers": 2 }, { "compute_ns": 7338333, "compute_span_ns": 7338333, "calibration_cpu_ns": 8177791, "calibration_gpu_ticks": 8177791, "compute_passes": 1, "command_buffer_ns": 7338374.91134, "command_buffers": 2 }, { "compute_ns": 7366500, "compute_span_ns": 7366500, "calibration_cpu_ns": 8099042, "calibration_gpu_ticks": 8099042, "compute_passes": 1, "command_buffer_ns": 7366541.65667, "command_buffers": 2 }, { "compute_ns": 7317750, "compute_span_ns": 7317750, "calibration_cpu_ns": 8166041, "calibration_gpu_ticks": 8166041, "compute_passes": 1, "command_buffer_ns": 7317833.31139, "command_buffers": 2 }, { "compute_ns": 7490667, "compute_span_ns": 7490667, "calibration_cpu_ns": 8610042, "calibration_gpu_ticks": 8610042, "compute_passes": 1, "command_buffer_ns": 7490833.28992, "command_buffers": 2 } ], "latency": [ { "compute_ns": 76334, "compute_span_ns": 76334, "calibration_cpu_ns": 819917, "calibration_gpu_ticks": 819917, "compute_passes": 1, "command_buffer_ns": 76458.3819546, "command_buffers": 2 }, { "compute_ns": 119042, "compute_span_ns": 119042, "calibration_cpu_ns": 1001500, "calibration_gpu_ticks": 1001500, "compute_passes": 1, "command_buffer_ns": 119208.300021, "command_buffers": 2 }, { "compute_ns": 89875, "compute_span_ns": 89875, "calibration_cpu_ns": 499833, "calibration_gpu_ticks": 499833, "compute_passes": 1, "command_buffer_ns": 89958.3101273, "command_buffers": 2 }, { "compute_ns": 122750, "compute_span_ns": 122750, "calibration_cpu_ns": 1150000, "calibration_gpu_ticks": 1150000, "compute_passes": 1, "command_buffer_ns": 122833.298519, "command_buffers": 2 }, { "compute_ns": 118458, "compute_span_ns": 118458, "calibration_cpu_ns": 1206125, "calibration_gpu_ticks": 1206125, "compute_passes": 1, "command_buffer_ns": 118416.675832, "command_buffers": 2 }, { "compute_ns": 77375, "compute_span_ns": 77375, "calibration_cpu_ns": 526917, "calibration_gpu_ticks": 526917, "compute_passes": 1, "command_buffer_ns": 77375.0361986, "command_buffers": 2 }, { "compute_ns": 116084, "compute_span_ns": 116084, "calibration_cpu_ns": 696125, "calibration_gpu_ticks": 696125, "compute_passes": 1, "command_buffer_ns": 116166.600492, "command_buffers": 2 }, { "compute_ns": 91958, "compute_span_ns": 91958, "calibration_cpu_ns": 479917, "calibration_gpu_ticks": 479917, "compute_passes": 1, "command_buffer_ns": 91958.3253562, "command_buffers": 2 }, { "compute_ns": 116250, "compute_span_ns": 116250, "calibration_cpu_ns": 521000, "calibration_gpu_ticks": 521000, "compute_passes": 1, "command_buffer_ns": 116333.307233, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7799916.60966, "command_buffers": 2 }, { "command_buffer_ns": 7413500.04682, "command_buffers": 2 }, { "command_buffer_ns": 7282708.36407, "command_buffers": 2 }, { "command_buffer_ns": 7418999.97229, "command_buffers": 2 }, { "command_buffer_ns": 7307541.61486, "command_buffers": 2 }, { "command_buffer_ns": 7382416.63203, "command_buffers": 2 }, { "command_buffer_ns": 7261375.02352, "command_buffers": 2 }, { "command_buffer_ns": 7285374.97343, "command_buffers": 2 }, { "command_buffer_ns": 7269291.67286, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 124416.663311, "command_buffers": 2 }, { "command_buffer_ns": 85375.0389069, "command_buffers": 2 }, { "command_buffer_ns": 121500.052046, "command_buffers": 2 }, { "command_buffer_ns": 120583.339594, "command_buffers": 2 }, { "command_buffer_ns": 76250.0567362, "command_buffers": 2 }, { "command_buffer_ns": 78458.3389759, "command_buffers": 2 }, { "command_buffer_ns": 352500.006557, "command_buffers": 2 }, { "command_buffer_ns": 76708.3256505, "command_buffers": 2 }, { "command_buffer_ns": 421291.682869, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 121.87369702593749, 115.8359382315625, 113.79231818859375, 115.92187456703125, 114.1803377321875, 115.35025987546875, 113.45898474250001, 113.83398395984375, 113.5826823884375 ], "command_buffer_throughput_us_p50": 114.1803377321875, "command_buffer_throughput_us_p90": 117.1122390588125, "command_buffer_latency_us": [ 124.416663311, 85.37503890689999, 121.500052046, 120.58333959400001, 76.2500567362, 78.45833897589999, 352.500006557, 76.7083256505, 421.291682869 ], "command_buffer_latency_us_p50": 120.58333959400001, "command_buffer_latency_us_p90": 366.2583418194 }, "compute_throughput_us": [ 114.0423125, 114.271484375, 113.622390625, 114.4329375, 113.7701875, 114.661453125, 115.1015625, 114.33984375, 117.041671875 ], "compute_throughput_us_p50": 114.33984375, "compute_throughput_us_p90": 115.489584375, "command_buffer_throughput_us": [ 114.04231827328125, 114.2714845628125, 113.62304667265624, 114.43294351921875, 113.77213559171875, 114.6621079896875, 115.10221338546876, 114.34114549046876, 117.044270155 ], "command_buffer_throughput_us_p50": 114.34114549046876, "command_buffer_throughput_us_p90": 115.49062473937501, "compute_latency_us": [ 76.334, 119.042, 89.875, 122.75, 118.458, 77.375, 116.084, 91.958, 116.25 ], "compute_latency_us_p50": 116.084, "compute_latency_us_p90": 119.7836, "command_buffer_latency_us": [ 76.4583819546, 119.208300021, 89.95831012730001, 122.83329851900001, 118.416675832, 77.3750361986, 116.166600492, 91.9583253562, 116.333307233 ], "command_buffer_latency_us_p50": 116.166600492, "command_buffer_latency_us_p90": 119.9332997206, "counter_control_throughput_ratio": 0.9964249349005588, "counter_control_latency_ratio": 0.9861926368638425 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 110.977257042, "throughput_us_p90": 112.622476761, "latency_us_p50": 344.791, "latency_us_p90": 919.3090000000004 }, "slowdown": 0.2913041942616523, "device_compute_slowdown": 0.3255902815766957, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-yioj1q21/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "6caff4b0a979c3104fea9f23f0d5430392f6f4e7d743dab03e9fdb6de1428bf3", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.65, "capture_ms": 0.082, "compile_ms": 64.52825, "allocation_upload_ms": 5.922541, "cold_call_ms": 6.639167, "warmup_ms": 200.578708, "download_ms": 3.171291, "repetitions": 467, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 90.2193062099, 90.4642226981, 90.7099400428, 91.182012848, 97.9660963597, 92.4304068522, 80.0113319058, 92.2483940043, 80.1978051392 ], "latency_us": [ 423.625, 397.709, 381, 2401.291, 600.042, 710.75, 409.792, 350.542, 340.833 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3922583, "compute_span_ns": 3922583, "calibration_cpu_ns": 4621000, "calibration_gpu_ticks": 4621000, "compute_passes": 1, "command_buffer_ns": 3922666.71406, "command_buffers": 2 }, { "compute_ns": 4527625, "compute_span_ns": 4527625, "calibration_cpu_ns": 5474500, "calibration_gpu_ticks": 5474500, "compute_passes": 1, "command_buffer_ns": 4527624.98753, "command_buffers": 2 }, { "compute_ns": 4666625, "compute_span_ns": 4666625, "calibration_cpu_ns": 5453666, "calibration_gpu_ticks": 5453666, "compute_passes": 1, "command_buffer_ns": 4666708.35158, "command_buffers": 2 }, { "compute_ns": 4699834, "compute_span_ns": 4699834, "calibration_cpu_ns": 5665417, "calibration_gpu_ticks": 5665417, "compute_passes": 1, "command_buffer_ns": 4699916.69524, "command_buffers": 2 }, { "compute_ns": 4596583, "compute_span_ns": 4596583, "calibration_cpu_ns": 5676458, "calibration_gpu_ticks": 5676458, "compute_passes": 1, "command_buffer_ns": 4596666.72396, "command_buffers": 2 }, { "compute_ns": 4405541, "compute_span_ns": 4405541, "calibration_cpu_ns": 5451208, "calibration_gpu_ticks": 5451208, "compute_passes": 1, "command_buffer_ns": 4405500.01804, "command_buffers": 2 }, { "compute_ns": 4710042, "compute_span_ns": 4710042, "calibration_cpu_ns": 5316459, "calibration_gpu_ticks": 5316459, "compute_passes": 1, "command_buffer_ns": 4710083.36172, "command_buffers": 2 }, { "compute_ns": 4601584, "compute_span_ns": 4601584, "calibration_cpu_ns": 5525083, "calibration_gpu_ticks": 5525083, "compute_passes": 1, "command_buffer_ns": 4601708.3223, "command_buffers": 2 }, { "compute_ns": 4684375, "compute_span_ns": 4684375, "calibration_cpu_ns": 6070625, "calibration_gpu_ticks": 6070625, "compute_passes": 1, "command_buffer_ns": 4684458.31211, "command_buffers": 2 } ], "latency": [ { "compute_ns": 95167, "compute_span_ns": 95167, "calibration_cpu_ns": 752625, "calibration_gpu_ticks": 752625, "compute_passes": 1, "command_buffer_ns": 95250.0267886, "command_buffers": 2 }, { "compute_ns": 106875, "compute_span_ns": 106875, "calibration_cpu_ns": 1122500, "calibration_gpu_ticks": 1122500, "compute_passes": 1, "command_buffer_ns": 106874.969788, "command_buffers": 2 }, { "compute_ns": 102584, "compute_span_ns": 102584, "calibration_cpu_ns": 566417, "calibration_gpu_ticks": 566417, "compute_passes": 1, "command_buffer_ns": 102666.672319, "command_buffers": 2 }, { "compute_ns": 63583, "compute_span_ns": 63583, "calibration_cpu_ns": 508916, "calibration_gpu_ticks": 508916, "compute_passes": 1, "command_buffer_ns": 63624.989707, "command_buffers": 2 }, { "compute_ns": 386166, "compute_span_ns": 386166, "calibration_cpu_ns": 1744542, "calibration_gpu_ticks": 1744542, "compute_passes": 1, "command_buffer_ns": 386166.677345, "command_buffers": 2 }, { "compute_ns": 88375, "compute_span_ns": 88375, "calibration_cpu_ns": 513875, "calibration_gpu_ticks": 513875, "compute_passes": 1, "command_buffer_ns": 88458.3569132, "command_buffers": 2 }, { "compute_ns": 425459, "compute_span_ns": 425459, "calibration_cpu_ns": 1284333, "calibration_gpu_ticks": 1284333, "compute_passes": 1, "command_buffer_ns": 425625.010394, "command_buffers": 2 }, { "compute_ns": 101167, "compute_span_ns": 101167, "calibration_cpu_ns": 612083, "calibration_gpu_ticks": 612083, "compute_passes": 1, "command_buffer_ns": 101249.95606, "command_buffers": 2 }, { "compute_ns": 567083, "compute_span_ns": 567083, "calibration_cpu_ns": 1357167, "calibration_gpu_ticks": 1357167, "compute_passes": 1, "command_buffer_ns": 567166.658584, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4373416.65383, "command_buffers": 2 }, { "command_buffer_ns": 4680208.33796, "command_buffers": 2 }, { "command_buffer_ns": 4621333.32621, "command_buffers": 2 }, { "command_buffer_ns": 4686541.68071, "command_buffers": 2 }, { "command_buffer_ns": 4677749.99561, "command_buffers": 2 }, { "command_buffer_ns": 4553791.65942, "command_buffers": 2 }, { "command_buffer_ns": 4488708.27677, "command_buffers": 2 }, { "command_buffer_ns": 4802250.01229, "command_buffers": 2 }, { "command_buffer_ns": 4057333.2808, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 172375.002876, "command_buffers": 2 }, { "command_buffer_ns": 105583.283585, "command_buffers": 2 }, { "command_buffer_ns": 511791.615281, "command_buffers": 2 }, { "command_buffer_ns": 63958.2867734, "command_buffers": 2 }, { "command_buffer_ns": 64583.3206363, "command_buffers": 2 }, { "command_buffer_ns": 66375.0106469, "command_buffers": 2 }, { "command_buffer_ns": 63416.6644886, "command_buffers": 2 }, { "command_buffer_ns": 63250.0159554, "command_buffers": 2 }, { "command_buffer_ns": 63541.6945443, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 68.33463521609376, 73.128255280625, 72.20833322203124, 73.22721376109375, 73.08984368140625, 71.15299467843751, 70.13606682453126, 75.03515644203125, 63.3958325125 ], "command_buffer_throughput_us_p50": 72.20833322203124, "command_buffer_throughput_us_p90": 73.58880229728125, "command_buffer_latency_us": [ 172.37500287600002, 105.583283585, 511.79161528099996, 63.958286773400005, 64.5833206363, 66.37501064690001, 63.4166644886, 63.250015955399995, 63.5416945443 ], "command_buffer_latency_us_p50": 64.5833206363, "command_buffer_latency_us_p90": 240.25832535700005 }, "compute_throughput_us": [ 61.290359375, 70.744140625, 72.916015625, 73.43490625, 71.821609375, 68.836578125, 73.59440625, 71.89975, 73.193359375 ], "compute_throughput_us_p50": 71.89975, "compute_throughput_us_p90": 73.46680625, "command_buffer_throughput_us": [ 61.291667407187504, 70.74414043015625, 72.91731799343749, 73.436198363125, 71.822917561875, 68.835937781875, 73.59505252687501, 71.9016925359375, 73.19466112671876 ], "command_buffer_throughput_us_p50": 71.9016925359375, "command_buffer_throughput_us_p90": 73.467969195875, "compute_latency_us": [ 95.167, 106.875, 102.584, 63.583, 386.166, 88.375, 425.459, 101.167, 567.083 ], "compute_latency_us_p50": 102.584, "compute_latency_us_p90": 453.78380000000004, "command_buffer_latency_us": [ 95.2500267886, 106.874969788, 102.666672319, 63.624989707, 386.166677345, 88.45835691319999, 425.62501039399996, 101.24995606, 567.166658584 ], "command_buffer_latency_us_p50": 102.666672319, "command_buffer_latency_us_p90": 453.933340032, "counter_control_throughput_ratio": 0.9826661810216247, "counter_control_latency_ratio": 1.3327057284220771 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 90.7099400428, "throughput_us_p90": 93.53754475369999, "latency_us_p50": 409.792, "latency_us_p90": 1048.8582000000004 }, "torch": { "cold_call_ms": 6.535917, "warmup_ms": 201.029667, "repetitions": 130, "throughput_us": [ 286.8666692307692, 287.94807692307694, 284.02819999999997, 280.3310923076923, 294.86474615384617, 287.04776153846154, 289.3285230769231, 301.38237692307695, 280.3884615384615 ], "latency_us": [ 712.4580000000001, 544.5, 530.083, 636.875, 2786.5829999999996, 1370.9579999999999, 677.333, 572.25, 565.8330000000001 ], "throughput_us_p50": 287.04776153846154, "throughput_us_p90": 296.16827230769235, "latency_us_p50": 636.875, "latency_us_p90": 1654.083, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17503499.98474121, "command_buffers": 1 }, { "command_buffer_ns": 15165249.991696328, "command_buffers": 1 }, { "command_buffer_ns": 17701750.039123, "command_buffers": 1 }, { "command_buffer_ns": 17457624.955568463, "command_buffers": 1 }, { "command_buffer_ns": 17493124.993052334, "command_buffers": 1 }, { "command_buffer_ns": 18403541.70417413, "command_buffers": 1 }, { "command_buffer_ns": 17938749.98111278, "command_buffers": 1 }, { "command_buffer_ns": 17510666.68657586, "command_buffers": 1 }, { "command_buffer_ns": 17885499.983094633, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 198625.02813339233, "command_buffers": 1 }, { "command_buffer_ns": 200500.01330673695, "command_buffers": 1 }, { "command_buffer_ns": 198625.02813339233, "command_buffers": 1 }, { "command_buffer_ns": 199625.0357478857, "command_buffers": 1 }, { "command_buffer_ns": 200750.0152103603, "command_buffers": 1 }, { "command_buffer_ns": 199250.00378862023, "command_buffers": 1 }, { "command_buffer_ns": 200625.04336237907, "command_buffers": 1 }, { "command_buffer_ns": 203666.6264757514, "command_buffers": 1 }, { "command_buffer_ns": 200624.98515471816, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 273.4921872615814, 236.95703112025512, 276.58984436129685, 272.7753899307572, 273.3300780164427, 287.5553391277208, 280.29296845488716, 273.6041669777478, 279.46093723585363 ], "command_buffer_throughput_us_p50": 273.6041669777478, "command_buffer_throughput_us_p90": 281.7454425894539, "command_buffer_latency_us": [ 198.62502813339233, 200.50001330673695, 198.62502813339233, 199.6250357478857, 200.7500152103603, 199.25000378862023, 200.62504336237907, 203.6666264757514, 200.62498515471816 ], "command_buffer_latency_us_p50": 200.50001330673695, "command_buffer_latency_us_p90": 201.3333374634385 }, "throughput": [ { "compute_ns": 17874458.0, "compute_span_ns": 17874458.0, "command_buffer_ns": 17874458.339065313, "calibration_cpu_ns": 20103125.0, "calibration_gpu_ticks": 20103125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17485042.0, "compute_span_ns": 17485042.0, "command_buffer_ns": 17485041.63697362, "calibration_cpu_ns": 18661000.0, "calibration_gpu_ticks": 18661000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17597833.0, "compute_span_ns": 17597833.0, "command_buffer_ns": 17597833.299078047, "calibration_cpu_ns": 20409292.0, "calibration_gpu_ticks": 20409292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17512958.0, "compute_span_ns": 17512958.0, "command_buffer_ns": 17512916.64550081, "calibration_cpu_ns": 20513584.0, "calibration_gpu_ticks": 20513584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17463209.0, "compute_span_ns": 17463209.0, "command_buffer_ns": 17463249.969296157, "calibration_cpu_ns": 19494417.0, "calibration_gpu_ticks": 19494417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17513958.0, "compute_span_ns": 17513958.0, "command_buffer_ns": 17513958.329800516, "calibration_cpu_ns": 19635375.0, "calibration_gpu_ticks": 19635375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17497458.0, "compute_span_ns": 17497458.0, "command_buffer_ns": 17497416.643891484, "calibration_cpu_ns": 18661333.0, "calibration_gpu_ticks": 18661333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17501875.0, "compute_span_ns": 17501875.0, "command_buffer_ns": 17501875.00147149, "calibration_cpu_ns": 18454083.0, "calibration_gpu_ticks": 18454083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17546500.0, "compute_span_ns": 17546500.0, "command_buffer_ns": 17546499.96291846, "calibration_cpu_ns": 19248167.0, "calibration_gpu_ticks": 19248167.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 201208.0, "compute_span_ns": 201208.0, "command_buffer_ns": 201166.6656471789, "calibration_cpu_ns": 893542.0, "calibration_gpu_ticks": 893542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 198625.0, "compute_span_ns": 198625.0, "command_buffer_ns": 198625.02813339233, "calibration_cpu_ns": 945500.0, "calibration_gpu_ticks": 945500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199625.0, "compute_span_ns": 199625.0, "command_buffer_ns": 199625.0357478857, "calibration_cpu_ns": 794667.0, "calibration_gpu_ticks": 794667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199000.0, "compute_span_ns": 199000.0, "command_buffer_ns": 199000.0018849969, "calibration_cpu_ns": 1053208.0, "calibration_gpu_ticks": 1053208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202625.0, "compute_span_ns": 202625.0, "command_buffer_ns": 202625.0003837049, "calibration_cpu_ns": 714083.0, "calibration_gpu_ticks": 714083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200584.0, "compute_span_ns": 200584.0, "command_buffer_ns": 200624.98515471816, "calibration_cpu_ns": 620250.0, "calibration_gpu_ticks": 620250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201875.0, "compute_span_ns": 201875.0, "command_buffer_ns": 201874.99467283487, "calibration_cpu_ns": 665917.0, "calibration_gpu_ticks": 665917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200041.0, "compute_span_ns": 200041.0, "command_buffer_ns": 200000.00949949026, "calibration_cpu_ns": 804084.0, "calibration_gpu_ticks": 804084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199333.0, "compute_span_ns": 199333.0, "command_buffer_ns": 199333.35715904832, "calibration_cpu_ns": 706542.0, "calibration_gpu_ticks": 706542.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 279.28840625, 273.20378125, 274.966140625, 273.63996875, 272.862640625, 273.65559375, 273.39778125, 273.466796875, 274.1640625 ], "compute_throughput_us_p50": 273.63996875, "compute_throughput_us_p90": 275.83059375, "command_buffer_throughput_us": [ 279.2884115478955, 273.2037755777128, 274.9661452980945, 273.63932258595014, 272.86328077025246, 273.65559890313307, 273.39713506080443, 273.466796897992, 274.16406192060094 ], "command_buffer_throughput_us_p50": 273.63932258595014, "command_buffer_throughput_us_p90": 275.8305985480547, "compute_latency_us": [ 201.208, 198.625, 199.625, 199.0, 202.625, 200.584, 201.875, 200.041, 199.333 ], "compute_latency_us_p50": 200.041, "compute_latency_us_p90": 202.025, "command_buffer_latency_us": [ 201.1666656471789, 198.62502813339233, 199.6250357478857, 199.0000018849969, 202.6250003837049, 200.62498515471816, 201.87499467283487, 200.00000949949026, 199.33335715904832 ], "command_buffer_latency_us_p50": 200.00000949949026, "command_buffer_latency_us_p90": 202.02499581500888, "counter_control_throughput_ratio": 0.9982921848573059, "counter_control_latency_ratio": 1.0050346505870438 }, "download_ms": 1.043084, "allocation_upload_ms": 18.427333, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3160099195918857, "device_compute_slowdown": 0.26275309973335176, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.574916, "warmup_ms": 200.799625, "repetitions": 105, "throughput_us": [ 299.4940476190476, 275.1972285714286, 297.4761904761905, 298.14682857142856, 274.4920666666667, 299.5392857142857, 251.61428571428573, 298.2059523809524, 284.66507619047616 ], "latency_us": [ 1592.666, 571.0, 477.166, 511.625, 423.375, 445.0, 431.25, 445.667, 427.333 ], "throughput_us_p50": 297.4761904761905, "throughput_us_p90": 299.5030952380952, "latency_us_p50": 445.667, "latency_us_p90": 775.3332000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17870250.04159659, "command_buffers": 1 }, { "command_buffer_ns": 17490791.622549295, "command_buffers": 1 }, { "command_buffer_ns": 17835083.359386772, "command_buffers": 1 }, { "command_buffer_ns": 17739500.03553182, "command_buffers": 1 }, { "command_buffer_ns": 17836166.662164032, "command_buffers": 1 }, { "command_buffer_ns": 17772375.023923814, "command_buffers": 1 }, { "command_buffer_ns": 17750833.299942315, "command_buffers": 1 }, { "command_buffer_ns": 17552874.982357025, "command_buffers": 1 }, { "command_buffer_ns": 17812874.98306483, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 198916.6485145688, "command_buffers": 1 }, { "command_buffer_ns": 221416.7034253478, "command_buffers": 1 }, { "command_buffer_ns": 199166.70862585306, "command_buffers": 1 }, { "command_buffer_ns": 200749.95700269938, "command_buffers": 1 }, { "command_buffer_ns": 200541.689991951, "command_buffers": 1 }, { "command_buffer_ns": 199999.95129182935, "command_buffers": 1 }, { "command_buffer_ns": 199875.03765150905, "command_buffers": 1 }, { "command_buffer_ns": 202249.96842443943, "command_buffers": 1 }, { "command_buffer_ns": 200875.04526600242, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 279.22265689994674, 273.29361910233274, 278.6731774904183, 277.17968805518467, 278.690104096313, 277.6933597488096, 277.35677031159867, 274.2636715993285, 278.326171610388 ], "command_buffer_throughput_us_p50": 277.6933597488096, "command_buffer_throughput_us_p90": 278.79661465703975, "command_buffer_latency_us": [ 198.9166485145688, 221.4167034253478, 199.16670862585306, 200.74995700269938, 200.541689991951, 199.99995129182935, 199.87503765150905, 202.24996842443943, 200.87504526600242 ], "command_buffer_latency_us_p50": 200.541689991951, "command_buffer_latency_us_p90": 206.0833154246211 }, "throughput": [ { "compute_ns": 17639292.0, "compute_span_ns": 17639292.0, "command_buffer_ns": 17639291.647356004, "calibration_cpu_ns": 18643959.0, "calibration_gpu_ticks": 18643959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17916708.0, "compute_span_ns": 17916708.0, "command_buffer_ns": 17916708.369739354, "calibration_cpu_ns": 19137541.0, "calibration_gpu_ticks": 19137541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17508291.0, "compute_span_ns": 17508291.0, "command_buffer_ns": 17508249.962702394, "calibration_cpu_ns": 18491375.0, "calibration_gpu_ticks": 18491375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17805834.0, "compute_span_ns": 17805834.0, "command_buffer_ns": 17805874.987971038, "calibration_cpu_ns": 18997416.0, "calibration_gpu_ticks": 18997416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17595000.0, "compute_span_ns": 17595000.0, "command_buffer_ns": 17594999.982975423, "calibration_cpu_ns": 18713542.0, "calibration_gpu_ticks": 18713542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17764875.0, "compute_span_ns": 17764875.0, "command_buffer_ns": 17764874.966815114, "calibration_cpu_ns": 20528042.0, "calibration_gpu_ticks": 20528042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17794042.0, "compute_span_ns": 17794042.0, "command_buffer_ns": 17794041.661545634, "calibration_cpu_ns": 18813917.0, "calibration_gpu_ticks": 18813917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17620208.0, "compute_span_ns": 17620208.0, "command_buffer_ns": 17620208.323933184, "calibration_cpu_ns": 20600542.0, "calibration_gpu_ticks": 20600542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17539791.0, "compute_span_ns": 17539791.0, "command_buffer_ns": 17539750.027935952, "calibration_cpu_ns": 18600375.0, "calibration_gpu_ticks": 18600375.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 198833.0, "compute_span_ns": 198833.0, "command_buffer_ns": 198833.35335180163, "calibration_cpu_ns": 855500.0, "calibration_gpu_ticks": 855500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201500.0, "compute_span_ns": 201500.0, "command_buffer_ns": 201499.9627135694, "calibration_cpu_ns": 622750.0, "calibration_gpu_ticks": 622750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200167.0, "compute_span_ns": 200167.0, "command_buffer_ns": 200208.33471789956, "calibration_cpu_ns": 874167.0, "calibration_gpu_ticks": 874167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202291.0, "compute_span_ns": 202291.0, "command_buffer_ns": 202250.02663210034, "calibration_cpu_ns": 709750.0, "calibration_gpu_ticks": 709750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200167.0, "compute_span_ns": 200167.0, "command_buffer_ns": 200208.33471789956, "calibration_cpu_ns": 594500.0, "calibration_gpu_ticks": 594500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199792.0, "compute_span_ns": 199792.0, "command_buffer_ns": 199833.3027586341, "calibration_cpu_ns": 608333.0, "calibration_gpu_ticks": 608333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200459.0, "compute_span_ns": 200459.0, "command_buffer_ns": 200500.01330673695, "calibration_cpu_ns": 635458.0, "calibration_gpu_ticks": 635458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199875.0, "compute_span_ns": 199875.0, "command_buffer_ns": 199874.97944384813, "calibration_cpu_ns": 640000.0, "calibration_gpu_ticks": 640000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199875.0, "compute_span_ns": 199875.0, "command_buffer_ns": 199874.97944384813, "calibration_cpu_ns": 683125.0, "calibration_gpu_ticks": 683125.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 275.6139375, 279.9485625, 273.567046875, 278.21615625, 274.921875, 277.576171875, 278.03190625, 275.31575, 274.059234375 ], "compute_throughput_us_p50": 275.6139375, "compute_throughput_us_p90": 278.5626375, "command_buffer_throughput_us": [ 275.61393198993756, 279.9485682771774, 273.5664056672249, 278.21679668704746, 274.921874733991, 277.57617135648616, 278.03190096165054, 275.315755061456, 274.05859418649925 ], "command_buffer_throughput_us_p50": 275.61393198993756, "command_buffer_throughput_us_p90": 278.56315100507345, "compute_latency_us": [ 198.833, 201.5, 200.167, 202.291, 200.167, 199.792, 200.459, 199.875, 199.875 ], "compute_latency_us_p50": 200.167, "compute_latency_us_p90": 201.6582, "command_buffer_latency_us": [ 198.83335335180163, 201.4999627135694, 200.20833471789956, 202.25002663210034, 200.20833471789956, 199.8333027586341, 200.50001330673695, 199.87497944384813, 199.87497944384813 ], "command_buffer_latency_us_p50": 200.20833471789956, "command_buffer_latency_us_p90": 201.6499754972756, "counter_control_throughput_ratio": 0.9995779935378021, "counter_control_latency_ratio": 0.999166757131095 }, "download_ms": 0.615, "allocation_upload_ms": 1.502708, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-slo6ygjk/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7d02ca31ce0660bb7b8cf2d4b7c68e2de8afe58cfb625e16c6aa93173ce8f234", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.589916, "capture_ms": 0.071125, "compile_ms": 64.6705, "allocation_upload_ms": 6.649583, "cold_call_ms": 8.393875, "warmup_ms": 200.208041, "download_ms": 2.846375, "repetitions": 289, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 112.089678201, 111.709775087, 112.100058824, 107.431228374, 106.21467474, 111.553633218, 112.399941176, 113.427913495, 112.930650519 ], "latency_us": [ 442.125, 376.458, 353.667, 467.583, 377, 365.5, 465.292, 4300.875, 364.417 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 5533875, "compute_span_ns": 5533875, "calibration_cpu_ns": 6341625, "calibration_gpu_ticks": 6341625, "compute_passes": 1, "command_buffer_ns": 5533916.64289, "command_buffers": 2 }, { "compute_ns": 5209000, "compute_span_ns": 5209000, "calibration_cpu_ns": 6100209, "calibration_gpu_ticks": 6100209, "compute_passes": 1, "command_buffer_ns": 5209124.99633, "command_buffers": 2 }, { "compute_ns": 5817125, "compute_span_ns": 5817125, "calibration_cpu_ns": 6827584, "calibration_gpu_ticks": 6827584, "compute_passes": 1, "command_buffer_ns": 5817249.99938, "command_buffers": 2 }, { "compute_ns": 5349167, "compute_span_ns": 5349167, "calibration_cpu_ns": 6582042, "calibration_gpu_ticks": 6582042, "compute_passes": 1, "command_buffer_ns": 5349208.30978, "command_buffers": 2 }, { "compute_ns": 5491500, "compute_span_ns": 5491500, "calibration_cpu_ns": 6487916, "calibration_gpu_ticks": 6487916, "compute_passes": 1, "command_buffer_ns": 5491500.02189, "command_buffers": 2 }, { "compute_ns": 5591291, "compute_span_ns": 5591291, "calibration_cpu_ns": 6677500, "calibration_gpu_ticks": 6677500, "compute_passes": 1, "command_buffer_ns": 5591374.93838, "command_buffers": 2 }, { "compute_ns": 5637708, "compute_span_ns": 5637708, "calibration_cpu_ns": 6525083, "calibration_gpu_ticks": 6525083, "compute_passes": 1, "command_buffer_ns": 5637749.97136, "command_buffers": 2 }, { "compute_ns": 5435417, "compute_span_ns": 5435417, "calibration_cpu_ns": 6465541, "calibration_gpu_ticks": 6465541, "compute_passes": 1, "command_buffer_ns": 5435458.38445, "command_buffers": 2 }, { "compute_ns": 7526166, "compute_span_ns": 7526166, "calibration_cpu_ns": 8307917, "calibration_gpu_ticks": 8307917, "compute_passes": 1, "command_buffer_ns": 7526208.35556, "command_buffers": 2 } ], "latency": [ { "compute_ns": 104000, "compute_span_ns": 104000, "calibration_cpu_ns": 515125, "calibration_gpu_ticks": 515125, "compute_passes": 1, "command_buffer_ns": 104083.33037, "command_buffers": 2 }, { "compute_ns": 79500, "compute_span_ns": 79500, "calibration_cpu_ns": 483417, "calibration_gpu_ticks": 483417, "compute_passes": 1, "command_buffer_ns": 79583.3184384, "command_buffers": 2 }, { "compute_ns": 99000, "compute_span_ns": 99000, "calibration_cpu_ns": 586958, "calibration_gpu_ticks": 586958, "compute_passes": 1, "command_buffer_ns": 99083.3505057, "command_buffers": 2 }, { "compute_ns": 77375, "compute_span_ns": 77375, "calibration_cpu_ns": 533125, "calibration_gpu_ticks": 533125, "compute_passes": 1, "command_buffer_ns": 77458.2731538, "command_buffers": 2 }, { "compute_ns": 109917, "compute_span_ns": 109917, "calibration_cpu_ns": 746583, "calibration_gpu_ticks": 746583, "compute_passes": 1, "command_buffer_ns": 110000.022687, "command_buffers": 2 }, { "compute_ns": 78709, "compute_span_ns": 78709, "calibration_cpu_ns": 634625, "calibration_gpu_ticks": 634625, "compute_passes": 1, "command_buffer_ns": 78833.3709352, "command_buffers": 2 }, { "compute_ns": 79625, "compute_span_ns": 79625, "calibration_cpu_ns": 709125, "calibration_gpu_ticks": 709125, "compute_passes": 1, "command_buffer_ns": 79708.348494, "command_buffers": 2 }, { "compute_ns": 115584, "compute_span_ns": 115584, "calibration_cpu_ns": 745209, "calibration_gpu_ticks": 745209, "compute_passes": 1, "command_buffer_ns": 115624.978207, "command_buffers": 2 }, { "compute_ns": 75875, "compute_span_ns": 75875, "calibration_cpu_ns": 632042, "calibration_gpu_ticks": 632042, "compute_passes": 1, "command_buffer_ns": 76000.0548325, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5406333.3082, "command_buffers": 2 }, { "command_buffer_ns": 5815291.60263, "command_buffers": 2 }, { "command_buffer_ns": 5514791.64278, "command_buffers": 2 }, { "command_buffer_ns": 5582333.36778, "command_buffers": 2 }, { "command_buffer_ns": 5354833.38172, "command_buffers": 2 }, { "command_buffer_ns": 5265583.2842, "command_buffers": 2 }, { "command_buffer_ns": 5801541.67255, "command_buffers": 2 }, { "command_buffer_ns": 5566749.9546, "command_buffers": 2 }, { "command_buffer_ns": 5287250.03824, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 509958.306793, "command_buffers": 2 }, { "command_buffer_ns": 85333.3040141, "command_buffers": 2 }, { "command_buffer_ns": 462041.702121, "command_buffers": 2 }, { "command_buffer_ns": 76791.679021, "command_buffers": 2 }, { "command_buffer_ns": 115708.331577, "command_buffers": 2 }, { "command_buffer_ns": 115000.002552, "command_buffers": 2 }, { "command_buffer_ns": 77041.6227169, "command_buffers": 2 }, { "command_buffer_ns": 83499.9955259, "command_buffers": 2 }, { "command_buffer_ns": 112833.33879, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 84.473957940625, 90.86393129109375, 86.1686194184375, 87.2239588715625, 83.669271589375, 82.274738815625, 90.64908863359375, 86.980468040625, 82.61328184749999 ], "command_buffer_throughput_us_p50": 86.1686194184375, "command_buffer_throughput_us_p90": 90.69205716509374, "command_buffer_latency_us": [ 509.958306793, 85.3333040141, 462.04170212099996, 76.79167902100001, 115.70833157700001, 115.00000255200001, 77.04162271690001, 83.4999955259, 112.83333879 ], "command_buffer_latency_us_p50": 112.83333879, "command_buffer_latency_us_p90": 471.6250230554 }, "compute_throughput_us": [ 86.466796875, 81.390625, 90.892578125, 83.580734375, 85.8046875, 87.363921875, 88.0891875, 84.928390625, 117.59634375 ], "compute_throughput_us_p50": 86.466796875, "compute_throughput_us_p90": 96.23333125, "command_buffer_throughput_us": [ 86.46744754515625, 81.39257806765626, 90.8945312403125, 83.5813798403125, 85.80468784203126, 87.36523341218751, 88.0898433025, 84.92903725703124, 117.597005555625 ], "command_buffer_throughput_us_p50": 86.46744754515625, "command_buffer_throughput_us_p90": 96.235026103375, "compute_latency_us": [ 104.0, 79.5, 99.0, 77.375, 109.917, 78.709, 79.625, 115.584, 75.875 ], "compute_latency_us_p50": 79.625, "compute_latency_us_p90": 111.0504, "command_buffer_latency_us": [ 104.08333037, 79.5833184384, 99.0833505057, 77.4582731538, 110.000022687, 78.8333709352, 79.708348494, 115.624978207, 76.0000548325 ], "command_buffer_latency_us_p50": 79.708348494, "command_buffer_latency_us_p90": 111.125013791, "counter_control_throughput_ratio": 1.0235988658887327, "counter_control_latency_ratio": 0.932617333383108 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 112.089678201, "throughput_us_p90": 113.0301031142, "latency_us_p50": 377.0, "latency_us_p90": 1234.2414000000008 }, "slowdown": 0.3768021838035857, "device_compute_slowdown": 0.31372432635051334, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-exi2gsii/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "d45e9e72129e03853b1c708b1bdcbc59be9547b0d0e02ca6bdc2eb0b41c08926", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.85825, "capture_ms": 0.06725, "compile_ms": 61.999125, "allocation_upload_ms": 1.004833, "cold_call_ms": 2.682792, "warmup_ms": 200.056917, "download_ms": 0.392458, "repetitions": 4029, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 6.38052246215, 6.50328865724, 6.78379647555, 6.27798461157, 6.24834524696, 6.33249565649, 6.93346140482, 6.34030354927, 6.17783569124 ], "latency_us": [ 247.542, 2301.667, 572.125, 425.334, 213.875, 196.333, 211.291, 194.584, 284.333 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 263709, "compute_span_ns": 263709, "calibration_cpu_ns": 711416, "calibration_gpu_ticks": 711416, "compute_passes": 1, "command_buffer_ns": 263791.705947, "command_buffers": 2 }, { "compute_ns": 712458, "compute_span_ns": 712458, "calibration_cpu_ns": 1416667, "calibration_gpu_ticks": 1416667, "compute_passes": 1, "command_buffer_ns": 712541.688699, "command_buffers": 2 }, { "compute_ns": 715625, "compute_span_ns": 715625, "calibration_cpu_ns": 1280625, "calibration_gpu_ticks": 1280625, "compute_passes": 1, "command_buffer_ns": 715708.360076, "command_buffers": 2 }, { "compute_ns": 276250, "compute_span_ns": 276250, "calibration_cpu_ns": 741125, "calibration_gpu_ticks": 741125, "compute_passes": 1, "command_buffer_ns": 276250.008028, "command_buffers": 2 }, { "compute_ns": 298042, "compute_span_ns": 298042, "calibration_cpu_ns": 824333, "calibration_gpu_ticks": 824333, "compute_passes": 1, "command_buffer_ns": 298083.352391, "command_buffers": 2 }, { "compute_ns": 264834, "compute_span_ns": 264834, "calibration_cpu_ns": 678291, "calibration_gpu_ticks": 678291, "compute_passes": 1, "command_buffer_ns": 264875.008725, "command_buffers": 2 }, { "compute_ns": 297750, "compute_span_ns": 297750, "calibration_cpu_ns": 753166, "calibration_gpu_ticks": 753166, "compute_passes": 1, "command_buffer_ns": 297791.73201, "command_buffers": 2 }, { "compute_ns": 708542, "compute_span_ns": 708542, "calibration_cpu_ns": 1951167, "calibration_gpu_ticks": 1951167, "compute_passes": 1, "command_buffer_ns": 708666.688297, "command_buffers": 2 }, { "compute_ns": 818042, "compute_span_ns": 818042, "calibration_cpu_ns": 1583666, "calibration_gpu_ticks": 1583666, "compute_passes": 1, "command_buffer_ns": 818125.030492, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6583, "compute_span_ns": 6583, "calibration_cpu_ns": 475750, "calibration_gpu_ticks": 475750, "compute_passes": 1, "command_buffer_ns": 6666.63981974, "command_buffers": 2 }, { "compute_ns": 197500, "compute_span_ns": 197500, "calibration_cpu_ns": 841584, "calibration_gpu_ticks": 841584, "compute_passes": 1, "command_buffer_ns": 197541.667148, "command_buffers": 2 }, { "compute_ns": 9833, "compute_span_ns": 9833, "calibration_cpu_ns": 473458, "calibration_gpu_ticks": 473458, "compute_passes": 1, "command_buffer_ns": 9833.31119642, "command_buffers": 2 }, { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 374083, "calibration_gpu_ticks": 374083, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 374625, "calibration_gpu_ticks": 374625, "compute_passes": 1, "command_buffer_ns": 6583.34465697, "command_buffers": 2 }, { "compute_ns": 9375, "compute_span_ns": 9375, "calibration_cpu_ns": 365875, "calibration_gpu_ticks": 365875, "compute_passes": 1, "command_buffer_ns": 9374.98407438, "command_buffers": 2 }, { "compute_ns": 8083, "compute_span_ns": 8083, "calibration_cpu_ns": 675250, "calibration_gpu_ticks": 675250, "compute_passes": 1, "command_buffer_ns": 8083.23966339, "command_buffers": 2 }, { "compute_ns": 7625, "compute_span_ns": 7625, "calibration_cpu_ns": 1232583, "calibration_gpu_ticks": 1232583, "compute_passes": 1, "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "compute_ns": 6916, "compute_span_ns": 6916, "calibration_cpu_ns": 534375, "calibration_gpu_ticks": 534375, "compute_passes": 1, "command_buffer_ns": 6916.64172336, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 447000.027634, "command_buffers": 2 }, { "command_buffer_ns": 279375.002719, "command_buffers": 2 }, { "command_buffer_ns": 263291.643932, "command_buffers": 2 }, { "command_buffer_ns": 295749.981888, "command_buffers": 2 }, { "command_buffer_ns": 832791.67302, "command_buffers": 2 }, { "command_buffer_ns": 592791.708186, "command_buffers": 2 }, { "command_buffer_ns": 266833.347268, "command_buffers": 2 }, { "command_buffer_ns": 265999.988187, "command_buffers": 2 }, { "command_buffer_ns": 263125.053607, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "command_buffer_ns": 8208.26971903, "command_buffers": 2 }, { "command_buffer_ns": 6625.02134219, "command_buffers": 2 }, { "command_buffer_ns": 88166.6783243, "command_buffers": 2 }, { "command_buffer_ns": 7791.67748988, "command_buffers": 2 }, { "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "command_buffer_ns": 175166.642293, "command_buffers": 2 }, { "command_buffer_ns": 108500.011265, "command_buffers": 2 }, { "command_buffer_ns": 7958.38423073, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 6.98437543178125, 4.365234417484375, 4.1139319364375, 4.621093467, 13.0123698909375, 9.26237044040625, 4.1692710510625, 4.156249815421875, 4.111328962609375 ], "command_buffer_throughput_us_p50": 4.365234417484375, "command_buffer_throughput_us_p90": 10.012370330512502, "command_buffer_latency_us": [ 6.54166797176, 8.208269719030001, 6.62502134219, 88.1666783243, 7.7916774898800005, 7.1667018346499995, 175.16664229300002, 108.500011265, 7.95838423073 ], "command_buffer_latency_us_p50": 7.95838423073, "command_buffer_latency_us_p90": 121.83333747060001 }, "compute_throughput_us": [ 4.120453125, 11.13215625, 11.181640625, 4.31640625, 4.65690625, 4.13803125, 4.65234375, 11.07096875, 12.78190625 ], "compute_throughput_us_p50": 4.65690625, "compute_throughput_us_p90": 11.501693750000001, "command_buffer_throughput_us": [ 4.121745405421875, 11.133463885921875, 11.1829431261875, 4.3164063754375, 4.657552381109375, 4.138672011328126, 4.65299581265625, 11.072917004640626, 12.783203601437501 ], "command_buffer_throughput_us_p50": 4.657552381109375, "command_buffer_throughput_us_p90": 11.5029952212375, "compute_latency_us": [ 6.583, 197.5, 9.833, 7.0, 6.5, 9.375, 8.083, 7.625, 6.916 ], "compute_latency_us_p50": 7.625, "compute_latency_us_p90": 47.366400000000034, "command_buffer_latency_us": [ 6.66663981974, 197.54166714800002, 9.83331119642, 6.99999509379, 6.58334465697, 9.37498407438, 8.08323966339, 7.66664743423, 6.91664172336 ], "command_buffer_latency_us_p50": 7.66664743423, "command_buffer_latency_us_p90": 47.374982386736036, "counter_control_throughput_ratio": 1.116021423330219, "counter_control_latency_ratio": 0.869101255082975 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 6.34030354927, "throughput_us_p90": 6.813729461404, "latency_us_p50": 247.542, "latency_us_p90": 918.0334000000003 }, "torch": { "cold_call_ms": 10.429584, "warmup_ms": 201.295584, "repetitions": 624, "throughput_us": [ 33.54360256410257, 34.80488782051282, 35.86885737179487, 32.52670993589743, 35.16386217948718, 32.814770833333334, 35.75801282051282, 95.05221634615384, 36.15344551282051 ], "latency_us": [ 342.792, 341.75, 311.542, 314.25, 317.70799999999997, 333.95799999999997, 347.917, 363.79200000000003, 378.125 ], "throughput_us_p50": 35.16386217948718, "throughput_us_p90": 47.93319967948719, "latency_us_p50": 341.75, "latency_us_p90": 366.65860000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1043416.6179038584, "command_buffers": 7 }, { "command_buffer_ns": 1032291.620504111, "command_buffers": 7 }, { "command_buffer_ns": 1070000.056643039, "command_buffers": 7 }, { "command_buffer_ns": 1030999.9925084412, "command_buffers": 7 }, { "command_buffer_ns": 1274666.6907332838, "command_buffers": 7 }, { "command_buffer_ns": 1025916.6592732072, "command_buffers": 7 }, { "command_buffer_ns": 1311999.9784976244, "command_buffers": 7 }, { "command_buffer_ns": 1122208.3121538162, "command_buffers": 7 }, { "command_buffer_ns": 1187124.98806417, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 70250.01104921103, "command_buffers": 1 }, { "command_buffer_ns": 57250.02847611904, "command_buffers": 1 }, { "command_buffer_ns": 57208.293583244085, "command_buffers": 1 }, { "command_buffer_ns": 70291.68773442507, "command_buffers": 1 }, { "command_buffer_ns": 70125.03920122981, "command_buffers": 1 }, { "command_buffer_ns": 86041.69124737382, "command_buffers": 1 }, { "command_buffer_ns": 88458.29870551825, "command_buffers": 1 }, { "command_buffer_ns": 69750.00724196434, "command_buffers": 1 }, { "command_buffer_ns": 76875.03239139915, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.303384654747788, 16.129556570376735, 16.718750885047484, 16.109374882944394, 19.91666704270756, 16.029947801143862, 20.49999966402538, 17.53450487740338, 18.548827938502654 ], "command_buffer_throughput_us_p50": 16.718750885047484, "command_buffer_throughput_us_p90": 20.033333566971123, "command_buffer_latency_us": [ 70.25001104921103, 57.25002847611904, 57.208293583244085, 70.29168773442507, 70.12503920122981, 86.04169124737382, 88.45829870551825, 69.75000724196434, 76.87503239139915 ], "command_buffer_latency_us_p50": 70.25001104921103, "command_buffer_latency_us_p90": 86.5250127390027 }, "throughput": [ { "compute_ns": 3846333.0, "compute_span_ns": 4740416.0, "command_buffer_ns": 4654874.908737838, "calibration_cpu_ns": 5543625.0, "calibration_gpu_ticks": 5543625.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3425875.0, "compute_span_ns": 4493792.0, "command_buffer_ns": 4412624.926771969, "calibration_cpu_ns": 6413000.0, "calibration_gpu_ticks": 6413000.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3668293.0, "compute_span_ns": 4897917.0, "command_buffer_ns": 4586625.145748258, "calibration_cpu_ns": 5875666.0, "calibration_gpu_ticks": 5875666.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3737958.0, "compute_span_ns": 5003875.0, "command_buffer_ns": 4736499.91909042, "calibration_cpu_ns": 6047792.0, "calibration_gpu_ticks": 6047792.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3628870.0, "compute_span_ns": 4703083.0, "command_buffer_ns": 4600875.137839466, "calibration_cpu_ns": 5625042.0, "calibration_gpu_ticks": 5625042.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3641455.0, "compute_span_ns": 4712708.0, "command_buffer_ns": 4601166.700012982, "calibration_cpu_ns": 5769917.0, "calibration_gpu_ticks": 5769917.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3813667.0, "compute_span_ns": 5257666.0, "command_buffer_ns": 4836583.335418254, "calibration_cpu_ns": 6305292.0, "calibration_gpu_ticks": 6305292.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3752453.0, "compute_span_ns": 4832250.0, "command_buffer_ns": 4706916.690338403, "calibration_cpu_ns": 5978584.0, "calibration_gpu_ticks": 5978584.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4030921.0, "compute_span_ns": 5176542.0, "command_buffer_ns": 5077416.659332812, "calibration_cpu_ns": 6130625.0, "calibration_gpu_ticks": 6130625.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 70250.0, "compute_span_ns": 70250.0, "command_buffer_ns": 70250.01104921103, "calibration_cpu_ns": 713750.0, "calibration_gpu_ticks": 713750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 57208.0, "compute_span_ns": 57208.0, "command_buffer_ns": 57166.675105690956, "calibration_cpu_ns": 524375.0, "calibration_gpu_ticks": 524375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70375.0, "compute_span_ns": 70375.0, "command_buffer_ns": 70374.98289719224, "calibration_cpu_ns": 696625.0, "calibration_gpu_ticks": 696625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71750.0, "compute_span_ns": 71750.0, "command_buffer_ns": 71750.02247095108, "calibration_cpu_ns": 925583.0, "calibration_gpu_ticks": 925583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 69750.0, "compute_span_ns": 69750.0, "command_buffer_ns": 69749.94903430343, "calibration_cpu_ns": 472583.0, "calibration_gpu_ticks": 472583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 54209.0, "compute_span_ns": 54209.0, "command_buffer_ns": 54250.00563263893, "calibration_cpu_ns": 516542.0, "calibration_gpu_ticks": 516542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70167.0, "compute_span_ns": 70167.0, "command_buffer_ns": 70166.65767878294, "calibration_cpu_ns": 912208.0, "calibration_gpu_ticks": 912208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 69875.0, "compute_span_ns": 69875.0, "command_buffer_ns": 69874.97908994555, "calibration_cpu_ns": 612000.0, "calibration_gpu_ticks": 612000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 73542.0, "compute_span_ns": 73542.0, "command_buffer_ns": 73541.6542738676, "calibration_cpu_ns": 486375.0, "calibration_gpu_ticks": 486375.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 60.098953125, 53.529296875, 57.317078125, 58.40559375, 56.70109375, 56.897734375, 59.588546875, 58.632078125, 62.983140625 ], "compute_throughput_us_p50": 58.40559375, "compute_throughput_us_p90": 60.675790625, "command_buffer_throughput_us": [ 72.73242044902872, 68.94726448081201, 71.66601790231653, 74.00781123578781, 71.88867402874166, 71.89322968770284, 75.57161461591022, 73.54557328653755, 79.33463530207518 ], "command_buffer_throughput_us_p50": 72.73242044902872, "command_buffer_throughput_us_p90": 76.32421875314321, "compute_latency_us": [ 70.25, 57.208, 70.375, 71.75, 69.75, 54.209, 70.167, 69.875, 73.542 ], "compute_latency_us_p50": 70.167, "compute_latency_us_p90": 72.1084, "command_buffer_latency_us": [ 70.25001104921103, 57.166675105690956, 70.37498289719224, 71.75002247095108, 69.74994903430343, 54.25000563263893, 70.16665767878294, 69.87497908994555, 73.54165427386761 ], "command_buffer_latency_us_p50": 70.16665767878294, "command_buffer_latency_us_p90": 72.10834883153439, "counter_control_throughput_ratio": 4.277069988740186, "counter_control_latency_ratio": 0.9985440466555776 }, "download_ms": 0.334917, "allocation_upload_ms": 0.745458, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.18030737115585135, "device_compute_slowdown": 0.07973390819265491, "valid": true, "round": 1, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.755208, "warmup_ms": 202.065292, "repetitions": 726, "throughput_us": [ 32.612546831955925, 34.797462809917356, 35.06887052341598, 36.73829201101928, 36.136307162534436, 32.551768595041324, 34.43864738292012, 33.163107438016524, 33.63756887052342 ], "latency_us": [ 380.125, 334.334, 346.08299999999997, 473.834, 336.75, 346.75, 344.875, 329.584, 332.91700000000003 ], "throughput_us_p50": 34.43864738292012, "throughput_us_p90": 36.256704132231405, "latency_us_p50": 344.875, "latency_us_p90": 398.8668, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1309124.8692944646, "command_buffers": 7 }, { "command_buffer_ns": 1047125.0279806554, "command_buffers": 7 }, { "command_buffer_ns": 1200833.3578705788, "command_buffers": 7 }, { "command_buffer_ns": 1132875.0988468528, "command_buffers": 7 }, { "command_buffer_ns": 1184499.8807646334, "command_buffers": 7 }, { "command_buffer_ns": 1334708.4168344736, "command_buffers": 7 }, { "command_buffer_ns": 1212291.6523367167, "command_buffers": 7 }, { "command_buffer_ns": 1323000.0622570515, "command_buffers": 7 }, { "command_buffer_ns": 1470833.3183079958, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 70458.33626762033, "command_buffers": 1 }, { "command_buffer_ns": 69416.71017557383, "command_buffers": 1 }, { "command_buffer_ns": 70874.98670443892, "command_buffers": 1 }, { "command_buffer_ns": 73208.29899981618, "command_buffers": 1 }, { "command_buffer_ns": 74166.68813675642, "command_buffers": 1 }, { "command_buffer_ns": 70000.00914558768, "command_buffers": 1 }, { "command_buffer_ns": 70375.04110485315, "command_buffers": 1 }, { "command_buffer_ns": 91166.64296016097, "command_buffers": 1 }, { "command_buffer_ns": 77208.32945778966, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 20.45507608272601, 16.36132856219774, 18.763021216727793, 17.701173419482075, 18.507810636947397, 20.85481901303865, 18.942057067761198, 20.67187597276643, 22.981770598562434 ], "command_buffer_throughput_us_p50": 18.942057067761198, "command_buffer_throughput_us_p90": 21.280209330143407, "command_buffer_latency_us": [ 70.45833626762033, 69.41671017557383, 70.87498670443892, 73.20829899981618, 74.16668813675642, 70.00000914558768, 70.37504110485315, 91.16664296016097, 77.20832945778966 ], "command_buffer_latency_us_p50": 70.87498670443892, "command_buffer_latency_us_p90": 79.99999215826392 }, "throughput": [ { "compute_ns": 3713370.0, "compute_span_ns": 4767042.0, "command_buffer_ns": 4669083.340559155, "calibration_cpu_ns": 5687209.0, "calibration_gpu_ticks": 5687209.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3704125.0, "compute_span_ns": 4764750.0, "command_buffer_ns": 4656833.421904594, "calibration_cpu_ns": 5762667.0, "calibration_gpu_ticks": 5762667.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3786502.0, "compute_span_ns": 4862917.0, "command_buffer_ns": 4775583.220180124, "calibration_cpu_ns": 5836583.0, "calibration_gpu_ticks": 5836583.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3816712.0, "compute_span_ns": 4781333.0, "command_buffer_ns": 4693124.967161566, "calibration_cpu_ns": 5802208.0, "calibration_gpu_ticks": 5802208.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4023252.0, "compute_span_ns": 5114791.0, "command_buffer_ns": 5001499.957870692, "calibration_cpu_ns": 6152833.0, "calibration_gpu_ticks": 6152833.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4119121.0, "compute_span_ns": 5276625.0, "command_buffer_ns": 5158041.6620709, "calibration_cpu_ns": 6347542.0, "calibration_gpu_ticks": 6347542.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4387496.0, "compute_span_ns": 5607708.0, "command_buffer_ns": 5490666.721016169, "calibration_cpu_ns": 6704125.0, "calibration_gpu_ticks": 6704125.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4446459.0, "compute_span_ns": 5533875.0, "command_buffer_ns": 5434124.963358045, "calibration_cpu_ns": 6592000.0, "calibration_gpu_ticks": 6592000.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4693250.0, "compute_span_ns": 5940792.0, "command_buffer_ns": 5812749.965116382, "calibration_cpu_ns": 7035000.0, "calibration_gpu_ticks": 7035000.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 69917.0, "compute_span_ns": 69917.0, "command_buffer_ns": 69916.6557751596, "calibration_cpu_ns": 571750.0, "calibration_gpu_ticks": 571750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 69875.0, "compute_span_ns": 69875.0, "command_buffer_ns": 69874.97908994555, "calibration_cpu_ns": 657791.0, "calibration_gpu_ticks": 657791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 86167.0, "compute_span_ns": 86167.0, "command_buffer_ns": 86208.33978056908, "calibration_cpu_ns": 769625.0, "calibration_gpu_ticks": 769625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88875.0, "compute_span_ns": 88875.0, "command_buffer_ns": 88875.00734999776, "calibration_cpu_ns": 539958.0, "calibration_gpu_ticks": 539958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 69791.0, "compute_span_ns": 69791.0, "command_buffer_ns": 69750.00724196434, "calibration_cpu_ns": 722625.0, "calibration_gpu_ticks": 722625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 69917.0, "compute_span_ns": 69917.0, "command_buffer_ns": 69958.33246037364, "calibration_cpu_ns": 757250.0, "calibration_gpu_ticks": 757250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88583.0, "compute_span_ns": 88583.0, "command_buffer_ns": 88541.65207594633, "calibration_cpu_ns": 549959.0, "calibration_gpu_ticks": 549959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 86750.0, "compute_span_ns": 86750.0, "command_buffer_ns": 86750.0202730298, "calibration_cpu_ns": 798958.0, "calibration_gpu_ticks": 798958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70375.0, "compute_span_ns": 70375.0, "command_buffer_ns": 70374.98289719224, "calibration_cpu_ns": 587542.0, "calibration_gpu_ticks": 587542.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 58.02140625, 57.876953125, 59.16409375, 59.636125, 62.8633125, 64.361265625, 68.554625, 69.475921875, 73.33203125 ], "compute_throughput_us_p50": 62.8633125, "compute_throughput_us_p90": 70.24714374999999, "command_buffer_throughput_us": [ 72.9544271962368, 72.76302221725928, 74.61848781531444, 73.33007761189947, 78.14843684172956, 80.59440096985782, 85.79166751587763, 84.90820255246945, 90.82421820494346 ], "command_buffer_throughput_us_p50": 78.14843684172956, "command_buffer_throughput_us_p90": 86.7981776536908, "compute_latency_us": [ 69.917, 69.875, 86.167, 88.875, 69.791, 69.917, 88.583, 86.75, 70.375 ], "compute_latency_us_p50": 70.375, "compute_latency_us_p90": 88.6414, "command_buffer_latency_us": [ 69.9166557751596, 69.87497908994555, 86.20833978056908, 88.87500734999776, 69.75000724196434, 69.95833246037364, 88.54165207594633, 86.7500202730298, 70.37498289719224 ], "command_buffer_latency_us_p50": 70.37498289719224, "command_buffer_latency_us_p90": 88.60832313075662, "counter_control_throughput_ratio": 4.107426082873626, "counter_control_latency_ratio": 0.9994046188604438 }, "download_ms": 0.312709, "allocation_upload_ms": 0.703709, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-1b7nt6ik/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "1ca0ba2a59ee9b200a0f7933e3c823dd873803c26a844f54bebdc043f01bc250", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.334334, "capture_ms": 0.067583, "compile_ms": 54.260042, "allocation_upload_ms": 1.075292, "cold_call_ms": 60.234625, "warmup_ms": 200.202625, "download_ms": 0.380917, "repetitions": 5101, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.41743462066, 4.39870437169, 4.78665947853, 4.30409057048, 4.37026230151, 4.33314546167, 4.26908939424, 4.35920250931, 4.2503838463 ], "latency_us": [ 808.708, 279.75, 346.875, 225.625, 257.584, 270.167, 234.042, 219, 252.209 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 266375, "compute_span_ns": 266375, "calibration_cpu_ns": 1047167, "calibration_gpu_ticks": 1047167, "compute_passes": 1, "command_buffer_ns": 266458.373517, "command_buffers": 2 }, { "compute_ns": 245000, "compute_span_ns": 245000, "calibration_cpu_ns": 819000, "calibration_gpu_ticks": 819000, "compute_passes": 1, "command_buffer_ns": 245000.002906, "command_buffers": 2 }, { "compute_ns": 242333, "compute_span_ns": 242333, "calibration_cpu_ns": 692083, "calibration_gpu_ticks": 692083, "compute_passes": 1, "command_buffer_ns": 242416.688707, "command_buffers": 2 }, { "compute_ns": 243500, "compute_span_ns": 243500, "calibration_cpu_ns": 967500, "calibration_gpu_ticks": 967500, "compute_passes": 1, "command_buffer_ns": 243583.344854, "command_buffers": 2 }, { "compute_ns": 262375, "compute_span_ns": 262375, "calibration_cpu_ns": 769250, "calibration_gpu_ticks": 769250, "compute_passes": 1, "command_buffer_ns": 262416.666374, "command_buffers": 2 }, { "compute_ns": 262209, "compute_span_ns": 262209, "calibration_cpu_ns": 716750, "calibration_gpu_ticks": 716750, "compute_passes": 1, "command_buffer_ns": 262291.636318, "command_buffers": 2 }, { "compute_ns": 283875, "compute_span_ns": 283875, "calibration_cpu_ns": 766458, "calibration_gpu_ticks": 766458, "compute_passes": 1, "command_buffer_ns": 283875.036985, "command_buffers": 2 }, { "compute_ns": 277291, "compute_span_ns": 277291, "calibration_cpu_ns": 960125, "calibration_gpu_ticks": 960125, "compute_passes": 1, "command_buffer_ns": 277291.63412, "command_buffers": 2 }, { "compute_ns": 269709, "compute_span_ns": 269709, "calibration_cpu_ns": 713333, "calibration_gpu_ticks": 713333, "compute_passes": 1, "command_buffer_ns": 269749.958534, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6709, "compute_span_ns": 6709, "calibration_cpu_ns": 423833, "calibration_gpu_ticks": 423833, "compute_passes": 1, "command_buffer_ns": 6833.3465606, "command_buffers": 2 }, { "compute_ns": 6834, "compute_span_ns": 6834, "calibration_cpu_ns": 781459, "calibration_gpu_ticks": 781459, "compute_passes": 1, "command_buffer_ns": 6958.31840858, "command_buffers": 2 }, { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 2177834, "calibration_gpu_ticks": 2177834, "compute_passes": 1, "command_buffer_ns": 6833.3465606, "command_buffers": 2 }, { "compute_ns": 6583, "compute_span_ns": 6583, "calibration_cpu_ns": 436916, "calibration_gpu_ticks": 436916, "compute_passes": 1, "command_buffer_ns": 6666.63981974, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 706667, "calibration_gpu_ticks": 706667, "compute_passes": 1, "command_buffer_ns": 7708.38232711, "command_buffers": 2 }, { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 503458, "calibration_gpu_ticks": 503458, "compute_passes": 1, "command_buffer_ns": 6749.93498251, "command_buffers": 2 }, { "compute_ns": 6667, "compute_span_ns": 6667, "calibration_cpu_ns": 413958, "calibration_gpu_ticks": 413958, "compute_passes": 1, "command_buffer_ns": 6708.31650496, "command_buffers": 2 }, { "compute_ns": 13625, "compute_span_ns": 13625, "calibration_cpu_ns": 480833, "calibration_gpu_ticks": 480833, "compute_passes": 1, "command_buffer_ns": 13708.3698064, "command_buffers": 2 }, { "compute_ns": 7542, "compute_span_ns": 7542, "calibration_cpu_ns": 600375, "calibration_gpu_ticks": 600375, "compute_passes": 1, "command_buffer_ns": 7625.02895668, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 266500.050202, "command_buffers": 2 }, { "command_buffer_ns": 249125.005212, "command_buffers": 2 }, { "command_buffer_ns": 242375.070229, "command_buffers": 2 }, { "command_buffer_ns": 242916.692514, "command_buffers": 2 }, { "command_buffer_ns": 296916.696243, "command_buffers": 2 }, { "command_buffer_ns": 269208.394457, "command_buffers": 2 }, { "command_buffer_ns": 242958.310992, "command_buffers": 2 }, { "command_buffer_ns": 244041.671976, "command_buffers": 2 }, { "command_buffer_ns": 263708.352577, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6708.31650496, "command_buffers": 2 }, { "command_buffer_ns": 6708.31650496, "command_buffers": 2 }, { "command_buffer_ns": 7750.05901232, "command_buffers": 2 }, { "command_buffer_ns": 13749.9300763, "command_buffers": 2 }, { "command_buffer_ns": 6749.99319017, "command_buffers": 2 }, { "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "command_buffer_ns": 6791.66987538, "command_buffers": 2 }, { "command_buffer_ns": 6624.96313453, "command_buffers": 2 }, { "command_buffer_ns": 6791.66987538, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.16406328440625, 3.8925782064375, 3.7871104723281253, 3.79557332053125, 4.639323378796875, 4.206381163390625, 3.79622360925, 3.813151124625, 4.1204430090156245 ], "command_buffer_throughput_us_p50": 3.8925782064375, "command_buffer_throughput_us_p90": 4.292969606471875, "command_buffer_latency_us": [ 6.70831650496, 6.70831650496, 7.75005901232, 13.7499300763, 6.7499931901700005, 6.99999509379, 6.79166987538, 6.624963134530001, 6.79166987538 ], "command_buffer_latency_us_p50": 6.79166987538, "command_buffer_latency_us_p90": 8.950033225116002 }, "compute_throughput_us": [ 4.162109375, 3.828125, 3.786453125, 3.8046875, 4.099609375, 4.097015625, 4.435546875, 4.332671875, 4.214203125 ], "compute_throughput_us_p50": 4.099609375, "compute_throughput_us_p90": 4.353246875, "command_buffer_throughput_us": [ 4.163412086203125, 3.8281250454062503, 3.787760761046875, 3.80598976334375, 4.100260412093751, 4.09830681746875, 4.435547452890625, 4.332681783125, 4.21484310209375 ], "command_buffer_throughput_us_p50": 4.100260412093751, "command_buffer_throughput_us_p90": 4.353254917078125, "compute_latency_us": [ 6.709, 6.834, 6.708, 6.583, 7.708, 6.708, 6.667, 13.625, 7.542 ], "compute_latency_us_p50": 6.709, "compute_latency_us_p90": 8.8914, "command_buffer_latency_us": [ 6.8333465606, 6.95831840858, 6.8333465606, 6.66663981974, 7.70838232711, 6.74993498251, 6.70831650496, 13.7083698064, 7.6250289566800005 ], "command_buffer_latency_us_p50": 6.8333465606, "command_buffer_latency_us_p90": 8.908379822968001, "counter_control_throughput_ratio": 1.0001717110508135, "counter_control_latency_ratio": 1.0186380674715563 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.35920250931, "throughput_us_p90": 4.491279592234, "latency_us_p50": 257.584, "latency_us_p90": 439.24160000000006 }, "slowdown": 0.12657879564317476, "device_compute_slowdown": 0.06521465719770972, "valid": true, "round": 1, "variant": "reference" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-rsuyegtw/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7f45b1ebfa78e24e3d477c2ad514fde5c49a3d7b63ed786757d9dc5d1979c8b1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.797083, "capture_ms": 0.068792, "compile_ms": 56.075292, "allocation_upload_ms": 6.068708, "cold_call_ms": 5.129541, "warmup_ms": 200.321166, "download_ms": 3.284917, "repetitions": 448, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 61.5440848214, 60.6872209821, 61.3724888393, 61.8630022321, 62.6038861607, 61.8493303571, 62.9162946429, 61.5627790179, 62.326265625 ], "latency_us": [ 665.042, 1109.208, 1732.083, 730.125, 388.25, 459.833, 531.084, 318.125, 291.583 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3580833, "compute_span_ns": 3580833, "calibration_cpu_ns": 4385375, "calibration_gpu_ticks": 4385375, "compute_passes": 1, "command_buffer_ns": 3580958.34963, "command_buffers": 2 }, { "compute_ns": 3592250, "compute_span_ns": 3592250, "calibration_cpu_ns": 4261625, "calibration_gpu_ticks": 4261625, "compute_passes": 1, "command_buffer_ns": 3592374.96741, "command_buffers": 2 }, { "compute_ns": 3650166, "compute_span_ns": 3650166, "calibration_cpu_ns": 4476167, "calibration_gpu_ticks": 4476167, "compute_passes": 1, "command_buffer_ns": 3650250.02975, "command_buffers": 2 }, { "compute_ns": 3644708, "compute_span_ns": 3644708, "calibration_cpu_ns": 4288875, "calibration_gpu_ticks": 4288875, "compute_passes": 1, "command_buffer_ns": 3644708.31119, "command_buffers": 2 }, { "compute_ns": 3602500, "compute_span_ns": 3602500, "calibration_cpu_ns": 4419708, "calibration_gpu_ticks": 4419708, "compute_passes": 1, "command_buffer_ns": 3602541.69209, "command_buffers": 2 }, { "compute_ns": 4130500, "compute_span_ns": 4130500, "calibration_cpu_ns": 4703250, "calibration_gpu_ticks": 4703250, "compute_passes": 1, "command_buffer_ns": 4130541.69621, "command_buffers": 2 }, { "compute_ns": 4167250, "compute_span_ns": 4167250, "calibration_cpu_ns": 4699083, "calibration_gpu_ticks": 4699083, "compute_passes": 1, "command_buffer_ns": 4167291.56859, "command_buffers": 2 }, { "compute_ns": 3615625, "compute_span_ns": 3615625, "calibration_cpu_ns": 4436417, "calibration_gpu_ticks": 4436417, "compute_passes": 1, "command_buffer_ns": 3615749.94167, "command_buffers": 2 }, { "compute_ns": 3658583, "compute_span_ns": 3658583, "calibration_cpu_ns": 4544583, "calibration_gpu_ticks": 4544583, "compute_passes": 1, "command_buffer_ns": 3658708.35958, "command_buffers": 2 } ], "latency": [ { "compute_ns": 71834, "compute_span_ns": 71834, "calibration_cpu_ns": 640458, "calibration_gpu_ticks": 640458, "compute_passes": 1, "command_buffer_ns": 71874.9943189, "command_buffers": 2 }, { "compute_ns": 60000, "compute_span_ns": 60000, "calibration_cpu_ns": 513291, "calibration_gpu_ticks": 513291, "compute_passes": 1, "command_buffer_ns": 60083.3445787, "command_buffers": 2 }, { "compute_ns": 59583, "compute_span_ns": 59583, "calibration_cpu_ns": 486708, "calibration_gpu_ticks": 486708, "compute_passes": 1, "command_buffer_ns": 59583.3407715, "command_buffers": 2 }, { "compute_ns": 62958, "compute_span_ns": 62958, "calibration_cpu_ns": 482875, "calibration_gpu_ticks": 482875, "compute_passes": 1, "command_buffer_ns": 63000.0140518, "command_buffers": 2 }, { "compute_ns": 67625, "compute_span_ns": 67625, "calibration_cpu_ns": 600042, "calibration_gpu_ticks": 600042, "compute_passes": 1, "command_buffer_ns": 67749.992013, "command_buffers": 2 }, { "compute_ns": 63917, "compute_span_ns": 63917, "calibration_cpu_ns": 522917, "calibration_gpu_ticks": 522917, "compute_passes": 1, "command_buffer_ns": 64041.6983515, "command_buffers": 2 }, { "compute_ns": 58708, "compute_span_ns": 58708, "calibration_cpu_ns": 576834, "calibration_gpu_ticks": 576834, "compute_passes": 1, "command_buffer_ns": 58708.3632126, "command_buffers": 2 }, { "compute_ns": 92208, "compute_span_ns": 92208, "calibration_cpu_ns": 789333, "calibration_gpu_ticks": 789333, "compute_passes": 1, "command_buffer_ns": 92249.9457374, "command_buffers": 2 }, { "compute_ns": 62250, "compute_span_ns": 62250, "calibration_cpu_ns": 510917, "calibration_gpu_ticks": 510917, "compute_passes": 1, "command_buffer_ns": 62374.9801889, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3740958.28738, "command_buffers": 2 }, { "command_buffer_ns": 3724375.0412, "command_buffers": 2 }, { "command_buffer_ns": 4131708.35236, "command_buffers": 2 }, { "command_buffer_ns": 4206833.31322, "command_buffers": 2 }, { "command_buffer_ns": 3607749.99717, "command_buffers": 2 }, { "command_buffer_ns": 3669583.29687, "command_buffers": 2 }, { "command_buffer_ns": 3600541.67686, "command_buffers": 2 }, { "command_buffer_ns": 3657791.64713, "command_buffers": 2 }, { "command_buffer_ns": 4121125.03545, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 95958.2976066, "command_buffers": 2 }, { "command_buffer_ns": 58625.0098422, "command_buffers": 2 }, { "command_buffer_ns": 58625.0098422, "command_buffers": 2 }, { "command_buffer_ns": 97041.7167991, "command_buffers": 2 }, { "command_buffer_ns": 61333.2958892, "command_buffers": 2 }, { "command_buffer_ns": 58999.9835938, "command_buffers": 2 }, { "command_buffer_ns": 59583.3407715, "command_buffers": 2 }, { "command_buffer_ns": 73041.6504666, "command_buffers": 2 }, { "command_buffer_ns": 66041.5971652, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 58.4524732403125, 58.19336001875, 64.55794300562499, 65.7317705190625, 56.37109370578125, 57.33723901359375, 56.2584637009375, 57.15299448640625, 64.39257867890625 ], "command_buffer_throughput_us_p50": 58.19336001875, "command_buffer_throughput_us_p90": 64.79270850831249, "command_buffer_latency_us": [ 95.9582976066, 58.6250098422, 58.6250098422, 97.0417167991, 61.3332958892, 58.9999835938, 59.5833407715, 73.04165046659999, 66.0415971652 ], "command_buffer_latency_us_p50": 61.3332958892, "command_buffer_latency_us_p90": 96.1749814451 }, "compute_throughput_us": [ 55.950515625, 56.12890625, 57.03384375, 56.9485625, 56.2890625, 64.5390625, 65.11328125, 56.494140625, 57.165359375 ], "compute_throughput_us_p50": 56.9485625, "compute_throughput_us_p90": 64.65390625, "command_buffer_throughput_us": [ 55.95247421296875, 56.13085886578125, 57.03515671484375, 56.948567362343745, 56.28971393890625, 64.53971400328125, 65.11393075921875, 56.49609283859375, 57.1673181184375 ], "command_buffer_throughput_us_p50": 56.948567362343745, "command_buffer_throughput_us_p90": 64.65455735446875, "compute_latency_us": [ 71.834, 60.0, 59.583, 62.958, 67.625, 63.917, 58.708, 92.208, 62.25 ], "compute_latency_us_p50": 62.958, "compute_latency_us_p90": 75.9088, "command_buffer_latency_us": [ 71.87499431890001, 60.0833445787, 59.5833407715, 63.000014051799994, 67.749992013, 64.0416983515, 58.7083632126, 92.24994573740001, 62.374980188900004 ], "command_buffer_latency_us_p50": 63.000014051799994, "command_buffer_latency_us_p90": 75.94998460260001, "counter_control_throughput_ratio": 0.9645577922927254, "counter_control_latency_ratio": 1.0163467934910293 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 61.8493303571, "throughput_us_p90": 62.66636785714, "latency_us_p50": 531.084, "latency_us_p90": 1233.7830000000001 }, "torch": { "cold_call_ms": 7.372375, "warmup_ms": 201.292166, "repetitions": 174, "throughput_us": [ 140.53735632183907, 140.58213218390804, 142.03113218390806, 140.37452298850573, 143.16522988505747, 141.76364942528735, 143.26005747126436, 142.41235632183907, 140.5941091954023 ], "latency_us": [ 775.625, 837.8330000000001, 607.333, 488.666, 431.458, 410.25, 402.33299999999997, 400.875, 430.375 ], "throughput_us_p50": 141.76364942528735, "throughput_us_p90": 143.18419540229885, "latency_us_p50": 431.458, "latency_us_p90": 788.0666, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 8499166.753608733, "command_buffers": 11 }, { "command_buffer_ns": 8451375.062577426, "command_buffers": 11 }, { "command_buffer_ns": 8443083.323072642, "command_buffers": 11 }, { "command_buffer_ns": 8399166.632443666, "command_buffers": 11 }, { "command_buffer_ns": 8440166.65359959, "command_buffers": 11 }, { "command_buffer_ns": 8530125.02193451, "command_buffers": 11 }, { "command_buffer_ns": 8437125.070486218, "command_buffers": 11 }, { "command_buffer_ns": 8406875.072978437, "command_buffers": 11 }, { "command_buffer_ns": 8505875.011906028, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 154874.98603761196, "command_buffers": 1 }, { "command_buffer_ns": 135458.36554840207, "command_buffers": 1 }, { "command_buffer_ns": 138500.0068694353, "command_buffers": 1 }, { "command_buffer_ns": 136874.96539205313, "command_buffers": 1 }, { "command_buffer_ns": 137666.7059957981, "command_buffers": 1 }, { "command_buffer_ns": 152541.6737422347, "command_buffers": 1 }, { "command_buffer_ns": 137083.29061046243, "command_buffers": 1 }, { "command_buffer_ns": 140458.345413208, "command_buffers": 1 }, { "command_buffer_ns": 172041.64760187268, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 132.79948052513646, 132.0527353527723, 131.92317692301003, 131.2369786319323, 131.8776039624936, 133.2832034677267, 131.83007922634715, 131.35742301528808, 132.90429706103168 ], "command_buffer_throughput_us_p50": 131.92317692301003, "command_buffer_throughput_us_p90": 132.9800783423707, "command_buffer_latency_us": [ 154.87498603761196, 135.45836554840207, 138.5000068694353, 136.87496539205313, 137.6667059957981, 152.5416737422347, 137.08329061046243, 140.458345413208, 172.04164760187268 ], "command_buffer_latency_us_p50": 138.5000068694353, "command_buffer_latency_us_p90": 158.3083183504641 }, "throughput": [ { "compute_ns": 9480960.0, "compute_span_ns": 10938625.0, "command_buffer_ns": 10686166.64269939, "calibration_cpu_ns": 12150583.0, "calibration_gpu_ticks": 12150583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9208122.0, "compute_span_ns": 10690041.0, "command_buffer_ns": 10448000.044561923, "calibration_cpu_ns": 12057458.0, "calibration_gpu_ticks": 12057458.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9328543.0, "compute_span_ns": 10777792.0, "command_buffer_ns": 10527916.660066694, "calibration_cpu_ns": 11767667.0, "calibration_gpu_ticks": 11767667.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9337958.0, "compute_span_ns": 10880458.0, "command_buffer_ns": 10634999.896865338, "calibration_cpu_ns": 11871125.0, "calibration_gpu_ticks": 11871125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9414372.0, "compute_span_ns": 10875334.0, "command_buffer_ns": 10636250.081006438, "calibration_cpu_ns": 12045584.0, "calibration_gpu_ticks": 12045584.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9325374.0, "compute_span_ns": 10914875.0, "command_buffer_ns": 10647833.172697574, "calibration_cpu_ns": 11969167.0, "calibration_gpu_ticks": 11969167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9206459.0, "compute_span_ns": 10773667.0, "command_buffer_ns": 10539666.63312167, "calibration_cpu_ns": 11963459.0, "calibration_gpu_ticks": 11963459.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9404458.0, "compute_span_ns": 10910959.0, "command_buffer_ns": 10685624.96220693, "calibration_cpu_ns": 12035792.0, "calibration_gpu_ticks": 12035792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9175123.0, "compute_span_ns": 10642416.0, "command_buffer_ns": 10377583.093941212, "calibration_cpu_ns": 11678500.0, "calibration_gpu_ticks": 11678500.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 147959.0, "compute_span_ns": 147959.0, "command_buffer_ns": 148000.0209994614, "calibration_cpu_ns": 781500.0, "calibration_gpu_ticks": 781500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138209.0, "compute_span_ns": 138209.0, "command_buffer_ns": 138250.00496581197, "calibration_cpu_ns": 606667.0, "calibration_gpu_ticks": 606667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 159416.0, "compute_span_ns": 159416.0, "command_buffer_ns": 159375.02030283213, "calibration_cpu_ns": 1596834.0, "calibration_gpu_ticks": 1596834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 148291.0, "compute_span_ns": 148291.0, "command_buffer_ns": 148250.02290308475, "calibration_cpu_ns": 618500.0, "calibration_gpu_ticks": 618500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135542.0, "compute_span_ns": 135542.0, "command_buffer_ns": 135541.66071116924, "calibration_cpu_ns": 593083.0, "calibration_gpu_ticks": 593083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 157666.0, "compute_span_ns": 157666.0, "command_buffer_ns": 157625.00697746873, "calibration_cpu_ns": 1316458.0, "calibration_gpu_ticks": 1316458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 141167.0, "compute_span_ns": 141167.0, "command_buffer_ns": 141166.674438864, "calibration_cpu_ns": 650208.0, "calibration_gpu_ticks": 650208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138166.0, "compute_span_ns": 138166.0, "command_buffer_ns": 138125.03311783075, "calibration_cpu_ns": 686833.0, "calibration_gpu_ticks": 686833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 151292.0, "compute_span_ns": 151292.0, "command_buffer_ns": 151291.664224118, "calibration_cpu_ns": 860625.0, "calibration_gpu_ticks": 860625.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 148.14, 143.87690625, 145.758484375, 145.90559375, 147.0995625, 145.70896875, 143.850921875, 146.94465625, 143.361296875 ], "compute_throughput_us_p50": 145.758484375, "compute_throughput_us_p90": 147.30765, "command_buffer_throughput_us": [ 166.97135379217798, 163.25000069628004, 164.4986978135421, 166.1718733885209, 166.1914075157256, 166.3723933233996, 164.68229114252608, 166.96289003448328, 162.14973584283143 ], "command_buffer_throughput_us_p50": 166.1718733885209, "command_buffer_throughput_us_p90": 166.96458278602222, "compute_latency_us": [ 147.959, 138.209, 159.416, 148.291, 135.542, 157.666, 141.167, 138.166, 151.292 ], "compute_latency_us_p50": 147.959, "compute_latency_us_p90": 158.016, "command_buffer_latency_us": [ 148.0000209994614, 138.25000496581197, 159.37502030283213, 148.25002290308475, 135.54166071116924, 157.62500697746873, 141.166674438864, 138.12503311783075, 151.291664224118 ], "command_buffer_latency_us_p50": 148.0000209994614, "command_buffer_latency_us_p90": 157.9750096425414, "counter_control_throughput_ratio": 1.2492011846535642, "counter_control_latency_ratio": 1.0206088373066364 }, "download_ms": 0.606625, "allocation_upload_ms": 16.667333, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.43628483470789875, "device_compute_slowdown": 0.3907049578910662, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.696209, "warmup_ms": 201.642833, "repetitions": 186, "throughput_us": [ 140.6478494623656, 139.60035483870968, 139.33870967741936, 139.15098387096776, 143.26187634408603, 142.84050537634408, 139.3709677419355, 141.4540806451613, 138.827064516129 ], "latency_us": [ 564.2909999999999, 420.25, 401.125, 430.084, 421.58299999999997, 416.708, 539.625, 668.5419999999999, 935.5 ], "throughput_us_p50": 139.60035483870968, "throughput_us_p90": 142.92477956989248, "latency_us_p50": 430.084, "latency_us_p90": 721.9336, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 8392624.964471906, "command_buffers": 11 }, { "command_buffer_ns": 8385916.764382273, "command_buffers": 11 }, { "command_buffer_ns": 8404375.053942204, "command_buffers": 11 }, { "command_buffer_ns": 8422750.048339367, "command_buffers": 11 }, { "command_buffer_ns": 8424624.97530505, "command_buffers": 11 }, { "command_buffer_ns": 8413958.363234997, "command_buffers": 11 }, { "command_buffer_ns": 8538291.67317599, "command_buffers": 11 }, { "command_buffer_ns": 8396375.051233917, "command_buffers": 11 }, { "command_buffer_ns": 8363166.7657755315, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 151625.01949816942, "command_buffers": 1 }, { "command_buffer_ns": 136499.99164044857, "command_buffers": 1 }, { "command_buffer_ns": 137874.9730065465, "command_buffers": 1 }, { "command_buffer_ns": 169250.0081844628, "command_buffers": 1 }, { "command_buffer_ns": 141499.9715052545, "command_buffers": 1 }, { "command_buffer_ns": 135916.6926704347, "command_buffers": 1 }, { "command_buffer_ns": 134749.97831508517, "command_buffers": 1 }, { "command_buffer_ns": 138916.6573062539, "command_buffers": 1 }, { "command_buffer_ns": 140124.99013915658, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 131.13476506987354, 131.02994944347301, 131.31836021784693, 131.6054695053026, 131.63476523914142, 131.46809942554682, 133.41080739337485, 131.19336017552996, 130.67448071524268 ], "command_buffer_throughput_us_p50": 131.31836021784693, "command_buffer_throughput_us_p90": 131.9899736699881, "command_buffer_latency_us": [ 151.62501949816942, 136.49999164044857, 137.8749730065465, 169.25000818446279, 141.4999715052545, 135.9166926704347, 134.74997831508517, 138.9166573062539, 140.12499013915658 ], "command_buffer_latency_us_p50": 138.9166573062539, "command_buffer_latency_us_p90": 155.1500172354281 }, "throughput": [ { "compute_ns": 9106374.0, "compute_span_ns": 10602208.0, "command_buffer_ns": 10328458.389267325, "calibration_cpu_ns": 11537416.0, "calibration_gpu_ticks": 11537416.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9096831.0, "compute_span_ns": 10583583.0, "command_buffer_ns": 10337083.251215518, "calibration_cpu_ns": 11574083.0, "calibration_gpu_ticks": 11574083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9097334.0, "compute_span_ns": 10556208.0, "command_buffer_ns": 10317916.807252914, "calibration_cpu_ns": 11491958.0, "calibration_gpu_ticks": 11491958.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9345548.0, "compute_span_ns": 10706583.0, "command_buffer_ns": 10502708.377316594, "calibration_cpu_ns": 11702583.0, "calibration_gpu_ticks": 11702583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9059543.0, "compute_span_ns": 10564834.0, "command_buffer_ns": 10326291.725505143, "calibration_cpu_ns": 11576708.0, "calibration_gpu_ticks": 11576708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9102460.0, "compute_span_ns": 10630500.0, "command_buffer_ns": 10419333.353638649, "calibration_cpu_ns": 11653041.0, "calibration_gpu_ticks": 11653041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9096376.0, "compute_span_ns": 10604458.0, "command_buffer_ns": 10346916.678827256, "calibration_cpu_ns": 11848125.0, "calibration_gpu_ticks": 11848125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9137995.0, "compute_span_ns": 10484666.0, "command_buffer_ns": 10255583.270918578, "calibration_cpu_ns": 11721916.0, "calibration_gpu_ticks": 11721916.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 9128920.0, "compute_span_ns": 10689167.0, "command_buffer_ns": 10423041.705507785, "calibration_cpu_ns": 11688250.0, "calibration_gpu_ticks": 11688250.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 140916.0, "compute_span_ns": 140916.0, "command_buffer_ns": 140874.9958500266, "calibration_cpu_ns": 682584.0, "calibration_gpu_ticks": 682584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136583.0, "compute_span_ns": 136583.0, "command_buffer_ns": 136541.6683256626, "calibration_cpu_ns": 662583.0, "calibration_gpu_ticks": 662583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135417.0, "compute_span_ns": 135417.0, "command_buffer_ns": 135458.36554840207, "calibration_cpu_ns": 636625.0, "calibration_gpu_ticks": 636625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138875.0, "compute_span_ns": 138875.0, "command_buffer_ns": 138875.03882870078, "calibration_cpu_ns": 593458.0, "calibration_gpu_ticks": 593458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 142958.0, "compute_span_ns": 142958.0, "command_buffer_ns": 142958.30624178052, "calibration_cpu_ns": 671417.0, "calibration_gpu_ticks": 671417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 140334.0, "compute_span_ns": 140334.0, "command_buffer_ns": 140374.99204277992, "calibration_cpu_ns": 651334.0, "calibration_gpu_ticks": 651334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137833.0, "compute_span_ns": 137833.0, "command_buffer_ns": 137791.67784377933, "calibration_cpu_ns": 699417.0, "calibration_gpu_ticks": 699417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 153709.0, "compute_span_ns": 153709.0, "command_buffer_ns": 153749.94836747646, "calibration_cpu_ns": 899292.0, "calibration_gpu_ticks": 899292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 159291.0, "compute_span_ns": 159291.0, "command_buffer_ns": 159249.99024719, "calibration_cpu_ns": 709625.0, "calibration_gpu_ticks": 709625.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 142.28709375, 142.137984375, 142.14584375, 146.0241875, 141.555359375, 142.2259375, 142.130875, 142.781171875, 142.639375 ], "compute_throughput_us_p50": 142.2259375, "compute_throughput_us_p90": 143.429775, "command_buffer_throughput_us": [ 161.38216233230196, 161.51692580024246, 161.21745011332678, 164.10481839557178, 161.34830821101787, 162.8020836506039, 161.67057310667587, 160.24348860810278, 162.86002664855914 ], "command_buffer_throughput_us_p50": 161.51692580024246, "command_buffer_throughput_us_p90": 163.10898499796167, "compute_latency_us": [ 140.916, 136.583, 135.417, 138.875, 142.958, 140.334, 137.833, 153.709, 159.291 ], "compute_latency_us_p50": 140.334, "compute_latency_us_p90": 154.8254, "command_buffer_latency_us": [ 140.8749958500266, 136.5416683256626, 135.45836554840207, 138.87503882870078, 142.95830624178052, 140.37499204277992, 137.79167784377933, 153.74994836747646, 159.24999024719 ], "command_buffer_latency_us_p50": 140.37499204277992, "command_buffer_latency_us_p90": 154.84995674341917, "counter_control_throughput_ratio": 1.2306588740698279, "counter_control_latency_ratio": 1.0103062546303896 }, "download_ms": 0.649708, "allocation_upload_ms": 0.903458, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-_xq6v1uq/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "5e168786762f89acc561194a669a3725da4fe77bed5800b1d573576d096bb3fc", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.54575, "capture_ms": 0.087208, "compile_ms": 53.446041, "allocation_upload_ms": 6.457625, "cold_call_ms": 7.234041, "warmup_ms": 200.957542, "download_ms": 3.810375, "repetitions": 461, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 63.795824295, 64.215472885, 63.8664143167, 63.9351952278, 65.0895704989, 63.4912342733, 64.4941236443, 65.1065618221, 64.0964381779 ], "latency_us": [ 362.416, 293.25, 301.166, 353.625, 295.375, 297.833, 301.791, 303.542, 288.792 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4065500, "compute_span_ns": 4065500, "calibration_cpu_ns": 4692583, "calibration_gpu_ticks": 4692583, "compute_passes": 1, "command_buffer_ns": 4065625.0203, "command_buffers": 2 }, { "compute_ns": 4029167, "compute_span_ns": 4029167, "calibration_cpu_ns": 4616500, "calibration_gpu_ticks": 4616500, "compute_passes": 1, "command_buffer_ns": 4029250.06347, "command_buffers": 2 }, { "compute_ns": 4030541, "compute_span_ns": 4030541, "calibration_cpu_ns": 4697208, "calibration_gpu_ticks": 4697208, "compute_passes": 1, "command_buffer_ns": 4030541.63326, "command_buffers": 2 }, { "compute_ns": 4022917, "compute_span_ns": 4022917, "calibration_cpu_ns": 4737125, "calibration_gpu_ticks": 4737125, "compute_passes": 1, "command_buffer_ns": 4023041.69256, "command_buffers": 2 }, { "compute_ns": 4030583, "compute_span_ns": 4030583, "calibration_cpu_ns": 4628667, "calibration_gpu_ticks": 4628667, "compute_passes": 1, "command_buffer_ns": 4030708.34, "command_buffers": 2 }, { "compute_ns": 4053667, "compute_span_ns": 4053667, "calibration_cpu_ns": 4540667, "calibration_gpu_ticks": 4540667, "compute_passes": 1, "command_buffer_ns": 4053791.63567, "command_buffers": 2 }, { "compute_ns": 4022166, "compute_span_ns": 4022166, "calibration_cpu_ns": 4692792, "calibration_gpu_ticks": 4692792, "compute_passes": 1, "command_buffer_ns": 4022249.95196, "command_buffers": 2 }, { "compute_ns": 4084500, "compute_span_ns": 4084500, "calibration_cpu_ns": 4588208, "calibration_gpu_ticks": 4588208, "compute_passes": 1, "command_buffer_ns": 4084583.31367, "command_buffers": 2 }, { "compute_ns": 4037584, "compute_span_ns": 4037584, "calibration_cpu_ns": 4664208, "calibration_gpu_ticks": 4664208, "compute_passes": 1, "command_buffer_ns": 4037750.01178, "command_buffers": 2 } ], "latency": [ { "compute_ns": 98959, "compute_span_ns": 98959, "calibration_cpu_ns": 675666, "calibration_gpu_ticks": 675666, "compute_passes": 1, "command_buffer_ns": 99083.292298, "command_buffers": 2 }, { "compute_ns": 76792, "compute_span_ns": 76792, "calibration_cpu_ns": 551916, "calibration_gpu_ticks": 551916, "compute_passes": 1, "command_buffer_ns": 76958.3275542, "command_buffers": 2 }, { "compute_ns": 63917, "compute_span_ns": 63917, "calibration_cpu_ns": 481583, "calibration_gpu_ticks": 481583, "compute_passes": 1, "command_buffer_ns": 64000.0216663, "command_buffers": 2 }, { "compute_ns": 63084, "compute_span_ns": 63084, "calibration_cpu_ns": 479000, "calibration_gpu_ticks": 479000, "compute_passes": 1, "command_buffer_ns": 63166.662585, "command_buffers": 2 }, { "compute_ns": 76125, "compute_span_ns": 76125, "calibration_cpu_ns": 652792, "calibration_gpu_ticks": 652792, "compute_passes": 1, "command_buffer_ns": 76208.3218433, "command_buffers": 2 }, { "compute_ns": 65333, "compute_span_ns": 65333, "calibration_cpu_ns": 560500, "calibration_gpu_ticks": 560500, "compute_passes": 1, "command_buffer_ns": 65416.7379253, "command_buffers": 2 }, { "compute_ns": 63959, "compute_span_ns": 63959, "calibration_cpu_ns": 465542, "calibration_gpu_ticks": 465542, "compute_passes": 1, "command_buffer_ns": 64041.5819362, "command_buffers": 2 }, { "compute_ns": 67333, "compute_span_ns": 67333, "calibration_cpu_ns": 786959, "calibration_gpu_ticks": 786959, "compute_passes": 1, "command_buffer_ns": 67333.3415762, "command_buffers": 2 }, { "compute_ns": 66250, "compute_span_ns": 66250, "calibration_cpu_ns": 521500, "calibration_gpu_ticks": 521500, "compute_passes": 1, "command_buffer_ns": 66291.6572765, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4083125.03714, "command_buffers": 2 }, { "command_buffer_ns": 4003374.95375, "command_buffers": 2 }, { "command_buffer_ns": 4024375.05545, "command_buffers": 2 }, { "command_buffer_ns": 4059291.67755, "command_buffers": 2 }, { "command_buffer_ns": 4013958.32887, "command_buffers": 2 }, { "command_buffer_ns": 4010125.00515, "command_buffers": 2 }, { "command_buffer_ns": 4029375.03532, "command_buffers": 2 }, { "command_buffer_ns": 3973624.96005, "command_buffers": 2 }, { "command_buffer_ns": 4010625.00896, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 81999.9258965, "command_buffers": 2 }, { "command_buffer_ns": 68125.0239722, "command_buffers": 2 }, { "command_buffer_ns": 63708.2848698, "command_buffers": 2 }, { "command_buffer_ns": 64166.6701995, "command_buffers": 2 }, { "command_buffer_ns": 112499.925308, "command_buffers": 2 }, { "command_buffer_ns": 63291.6926406, "command_buffers": 2 }, { "command_buffer_ns": 63166.7207927, "command_buffers": 2 }, { "command_buffer_ns": 113374.961074, "command_buffers": 2 }, { "command_buffer_ns": 76166.7033657, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 63.7988287053125, 62.55273365234375, 62.880860241406246, 63.426432461718754, 62.71809888859375, 62.65820320546875, 62.958984926875, 62.08789000078125, 62.666015765 ], "command_buffer_throughput_us_p50": 62.71809888859375, "command_buffer_throughput_us_p90": 63.50091171043751, "command_buffer_latency_us": [ 81.9999258965, 68.1250239722, 63.7082848698, 64.16667019949999, 112.499925308, 63.2916926406, 63.1667207927, 113.37496107400001, 76.1667033657 ], "command_buffer_latency_us_p50": 68.1250239722, "command_buffer_latency_us_p90": 112.6749324612 }, "compute_throughput_us": [ 63.5234375, 62.955734375, 62.977203125, 62.858078125, 62.977859375, 63.338546875, 62.84634375, 63.8203125, 63.08725 ], "compute_throughput_us_p50": 62.977859375, "compute_throughput_us_p90": 63.5828125, "command_buffer_throughput_us": [ 63.5253909421875, 62.95703224171875, 62.977213019687504, 62.86002644625, 62.979817812499995, 63.340494307343754, 62.847655499375, 63.821614276093754, 63.0898439340625 ], "command_buffer_throughput_us_p50": 62.979817812499995, "command_buffer_throughput_us_p90": 63.58463560896875, "compute_latency_us": [ 98.959, 76.792, 63.917, 63.084, 76.125, 65.333, 63.959, 67.333, 66.25 ], "compute_latency_us_p50": 66.25, "compute_latency_us_p90": 81.22540000000001, "command_buffer_latency_us": [ 99.083292298, 76.95832755420001, 64.0000216663, 63.166662585, 76.2083218433, 65.4167379253, 64.0415819362, 67.3333415762, 66.2916572765 ], "command_buffer_latency_us_p50": 66.2916572765, "command_buffer_latency_us_p90": 81.38332050296002, "counter_control_throughput_ratio": 1.0041729409619247, "counter_control_latency_ratio": 1.00457925993607 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 64.0964381779, "throughput_us_p90": 65.09296876354, "latency_us_p50": 301.166, "latency_us_p90": 355.3832 }, "slowdown": 0.4591423728969401, "device_compute_slowdown": 0.44280150640596067, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-lb7f0n_f/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "117627a7b60c4cee3b9bce9542bc5f280f025f1ad809d52b8d22cb3cdd24d0d1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 21.10725, "capture_ms": 0.0675, "compile_ms": 55.151792, "allocation_upload_ms": 6.456583, "cold_call_ms": 10.113167, "warmup_ms": 200.103166, "download_ms": 3.350833, "repetitions": 508, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 54.2895354331, 54.4155177165, 54.2079232283, 54.3626141732, 54.1437814961, 54.9537401575, 54.4613681102, 55.1861870079, 54.7763287402 ], "latency_us": [ 403.834, 281.667, 300.125, 358.25, 365.417, 366.167, 303.416, 360, 272.959 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3569834, "compute_span_ns": 3569834, "calibration_cpu_ns": 4317541, "calibration_gpu_ticks": 4317541, "compute_passes": 1, "command_buffer_ns": 3569916.64739, "command_buffers": 2 }, { "compute_ns": 3599417, "compute_span_ns": 3599417, "calibration_cpu_ns": 4433833, "calibration_gpu_ticks": 4433833, "compute_passes": 1, "command_buffer_ns": 3599541.72745, "command_buffers": 2 }, { "compute_ns": 3432667, "compute_span_ns": 3432667, "calibration_cpu_ns": 4092375, "calibration_gpu_ticks": 4092375, "compute_passes": 1, "command_buffer_ns": 3432708.32673, "command_buffers": 2 }, { "compute_ns": 3460459, "compute_span_ns": 3460459, "calibration_cpu_ns": 3985000, "calibration_gpu_ticks": 3985000, "compute_passes": 1, "command_buffer_ns": 3460583.27705, "command_buffers": 2 }, { "compute_ns": 3500042, "compute_span_ns": 3500042, "calibration_cpu_ns": 3992208, "calibration_gpu_ticks": 3992208, "compute_passes": 1, "command_buffer_ns": 3500125.02167, "command_buffers": 2 }, { "compute_ns": 3394750, "compute_span_ns": 3394750, "calibration_cpu_ns": 4179250, "calibration_gpu_ticks": 4179250, "compute_passes": 1, "command_buffer_ns": 3394750.0051, "command_buffers": 2 }, { "compute_ns": 3537459, "compute_span_ns": 3537459, "calibration_cpu_ns": 4071375, "calibration_gpu_ticks": 4071375, "compute_passes": 1, "command_buffer_ns": 3537583.33949, "command_buffers": 2 }, { "compute_ns": 3364333, "compute_span_ns": 3364333, "calibration_cpu_ns": 4092250, "calibration_gpu_ticks": 4092250, "compute_passes": 1, "command_buffer_ns": 3364333.30085, "command_buffers": 2 }, { "compute_ns": 3374916, "compute_span_ns": 3374916, "calibration_cpu_ns": 3983500, "calibration_gpu_ticks": 3983500, "compute_passes": 1, "command_buffer_ns": 3374916.67597, "command_buffers": 2 } ], "latency": [ { "compute_ns": 105167, "compute_span_ns": 105167, "calibration_cpu_ns": 757667, "calibration_gpu_ticks": 757667, "compute_passes": 1, "command_buffer_ns": 105250.044726, "command_buffers": 2 }, { "compute_ns": 69500, "compute_span_ns": 69500, "calibration_cpu_ns": 588459, "calibration_gpu_ticks": 588459, "compute_passes": 1, "command_buffer_ns": 69541.6820236, "command_buffers": 2 }, { "compute_ns": 54666, "compute_span_ns": 54666, "calibration_cpu_ns": 581458, "calibration_gpu_ticks": 581458, "compute_passes": 1, "command_buffer_ns": 54624.9793842, "command_buffers": 2 }, { "compute_ns": 62958, "compute_span_ns": 62958, "calibration_cpu_ns": 569417, "calibration_gpu_ticks": 569417, "compute_passes": 1, "command_buffer_ns": 63041.6325293, "command_buffers": 2 }, { "compute_ns": 59292, "compute_span_ns": 59292, "calibration_cpu_ns": 528208, "calibration_gpu_ticks": 528208, "compute_passes": 1, "command_buffer_ns": 59416.750446, "command_buffers": 2 }, { "compute_ns": 54042, "compute_span_ns": 54042, "calibration_cpu_ns": 487292, "calibration_gpu_ticks": 487292, "compute_passes": 1, "command_buffer_ns": 54166.7104699, "command_buffers": 2 }, { "compute_ns": 94958, "compute_span_ns": 94958, "calibration_cpu_ns": 719625, "calibration_gpu_ticks": 719625, "compute_passes": 1, "command_buffer_ns": 94916.6715145, "command_buffers": 2 }, { "compute_ns": 66250, "compute_span_ns": 66250, "calibration_cpu_ns": 553833, "calibration_gpu_ticks": 553833, "compute_passes": 1, "command_buffer_ns": 66291.6572765, "command_buffers": 2 }, { "compute_ns": 54208, "compute_span_ns": 54208, "calibration_cpu_ns": 445625, "calibration_gpu_ticks": 445625, "compute_passes": 1, "command_buffer_ns": 54208.2707398, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3597499.91912, "command_buffers": 2 }, { "command_buffer_ns": 3458499.90845, "command_buffers": 2 }, { "command_buffer_ns": 3483166.68533, "command_buffers": 2 }, { "command_buffer_ns": 3377666.69691, "command_buffers": 2 }, { "command_buffer_ns": 3478083.35209, "command_buffers": 2 }, { "command_buffer_ns": 3442374.98939, "command_buffers": 2 }, { "command_buffer_ns": 3438250.04529, "command_buffers": 2 }, { "command_buffer_ns": 3427666.67018, "command_buffers": 2 }, { "command_buffer_ns": 3454499.9944, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 61208.3240412, "command_buffers": 2 }, { "command_buffer_ns": 62375.0383966, "command_buffers": 2 }, { "command_buffer_ns": 58958.3069086, "command_buffers": 2 }, { "command_buffer_ns": 101291.690953, "command_buffers": 2 }, { "command_buffer_ns": 66875.0144541, "command_buffers": 2 }, { "command_buffer_ns": 53791.6785106, "command_buffers": 2 }, { "command_buffer_ns": 56375.0509173, "command_buffers": 2 }, { "command_buffer_ns": 54333.3590031, "command_buffers": 2 }, { "command_buffer_ns": 58041.6526645, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 56.21093623625001, 54.03906106953125, 54.42447945828125, 52.77604213921875, 54.34505237640625, 53.78710920921875, 53.72265695765625, 53.5572917215625, 53.9765624125 ], "command_buffer_throughput_us_p50": 53.9765624125, "command_buffer_throughput_us_p90": 54.781770813875, "command_buffer_latency_us": [ 61.2083240412, 62.375038396600004, 58.9583069086, 101.291690953, 66.87501445410001, 53.7916785106, 56.3750509173, 54.3333590031, 58.0416526645 ], "command_buffer_latency_us_p50": 58.9583069086, "command_buffer_latency_us_p90": 73.75834975388001 }, "compute_throughput_us": [ 55.77865625, 56.240890625, 53.635421875, 54.069671875, 54.68815625, 53.04296875, 55.272796875, 52.567703125, 52.7330625 ], "compute_throughput_us_p50": 54.069671875, "compute_throughput_us_p90": 55.871103125, "command_buffer_throughput_us": [ 55.77994761546875, 56.24283949140625, 53.63606760515625, 54.07161370390625, 54.68945346359375, 53.0429688296875, 55.27473967953125, 52.567707825781255, 52.73307306203125 ], "command_buffer_throughput_us_p50": 54.07161370390625, "command_buffer_throughput_us_p90": 55.87252599065625, "compute_latency_us": [ 105.167, 69.5, 54.666, 62.958, 59.292, 54.042, 94.958, 66.25, 54.208 ], "compute_latency_us_p50": 62.958, "compute_latency_us_p90": 96.99980000000001, "command_buffer_latency_us": [ 105.250044726, 69.5416820236, 54.6249793842, 63.041632529299996, 59.416750445999995, 54.166710469899996, 94.91667151450001, 66.2916572765, 54.2082707398 ], "command_buffer_latency_us_p50": 63.041632529299996, "command_buffer_latency_us_p90": 96.98334615680001, "counter_control_throughput_ratio": 0.9923326553578499, "counter_control_latency_ratio": 1.0069719326424456 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 54.4155177165, "throughput_us_p90": 55.000229527580004, "latency_us_p50": 358.25, "latency_us_p90": 373.7004 }, "torch": { "cold_call_ms": 4.421083, "warmup_ms": 200.681416, "repetitions": 229, "throughput_us": [ 129.05603930131005, 128.94832751091704, 130.61626637554585, 129.27292576419214, 128.63773362445414, 128.40393013100436, 128.44196069868994, 128.74963318777293, 129.65829694323145 ], "latency_us": [ 602.333, 407.292, 446.125, 735.709, 393.83299999999997, 392.95799999999997, 419.79100000000005, 494.875, 491.833 ], "throughput_us_p50": 128.94832751091704, "throughput_us_p90": 129.84989082969432, "latency_us_p50": 446.125, "latency_us_p90": 629.0082, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 8049916.70884192, "command_buffers": 11 }, { "command_buffer_ns": 7798208.273015916, "command_buffers": 11 }, { "command_buffer_ns": 7725041.708908975, "command_buffers": 11 }, { "command_buffer_ns": 7892041.525337845, "command_buffers": 11 }, { "command_buffer_ns": 7878458.360210061, "command_buffers": 11 }, { "command_buffer_ns": 7807749.9638311565, "command_buffers": 11 }, { "command_buffer_ns": 7738874.992355704, "command_buffers": 11 }, { "command_buffer_ns": 7806166.599038988, "command_buffers": 11 }, { "command_buffer_ns": 7758958.323393017, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 141249.96960163116, "command_buffers": 1 }, { "command_buffer_ns": 125916.61652550101, "command_buffers": 1 }, { "command_buffer_ns": 130000.00035390258, "command_buffers": 1 }, { "command_buffer_ns": 133583.32216739655, "command_buffers": 1 }, { "command_buffer_ns": 135875.01598522067, "command_buffers": 1 }, { "command_buffer_ns": 129333.34801346064, "command_buffers": 1 }, { "command_buffer_ns": 148041.63947701454, "command_buffers": 1 }, { "command_buffer_ns": 128749.99083578587, "command_buffers": 1 }, { "command_buffer_ns": 133083.31836014986, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 125.779948575655, 121.8470042658737, 120.70377670170274, 123.31314883340383, 123.1009118782822, 121.99609318486182, 120.91992175555788, 121.9713531099842, 121.23372380301589 ], "command_buffer_throughput_us_p50": 121.9713531099842, "command_buffer_throughput_us_p90": 123.80650878185406, "command_buffer_latency_us": [ 141.24996960163116, 125.91661652550101, 130.00000035390258, 133.58332216739655, 135.87501598522067, 129.33334801346064, 148.04163947701454, 128.74999083578587, 133.08331836014986 ], "command_buffer_latency_us_p50": 133.08331836014986, "command_buffer_latency_us_p90": 142.60830357670784 }, "throughput": [ { "compute_ns": 8106544.0, "compute_span_ns": 9315750.0, "command_buffer_ns": 9133041.60349071, "calibration_cpu_ns": 10625917.0, "calibration_gpu_ticks": 10625917.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8222204.0, "compute_span_ns": 9632333.0, "command_buffer_ns": 9397749.963682145, "calibration_cpu_ns": 10643209.0, "calibration_gpu_ticks": 10643209.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 7960045.0, "compute_span_ns": 9299250.0, "command_buffer_ns": 9069208.230357617, "calibration_cpu_ns": 10547250.0, "calibration_gpu_ticks": 10547250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8080499.0, "compute_span_ns": 9492833.0, "command_buffer_ns": 9261541.650630534, "calibration_cpu_ns": 10449625.0, "calibration_gpu_ticks": 10449625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8198452.0, "compute_span_ns": 9465875.0, "command_buffer_ns": 9277791.65795073, "calibration_cpu_ns": 10610875.0, "calibration_gpu_ticks": 10610875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8129457.0, "compute_span_ns": 9551042.0, "command_buffer_ns": 9330124.885309488, "calibration_cpu_ns": 10589500.0, "calibration_gpu_ticks": 10589500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8051253.0, "compute_span_ns": 9454292.0, "command_buffer_ns": 9235041.739884764, "calibration_cpu_ns": 10459667.0, "calibration_gpu_ticks": 10459667.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8319540.0, "compute_span_ns": 9562250.0, "command_buffer_ns": 9378208.254929632, "calibration_cpu_ns": 10583458.0, "calibration_gpu_ticks": 10583458.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8057336.0, "compute_span_ns": 9562791.0, "command_buffer_ns": 9337083.43654871, "calibration_cpu_ns": 10592750.0, "calibration_gpu_ticks": 10592750.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 145125.0, "compute_span_ns": 145125.0, "command_buffer_ns": 145125.02821162343, "calibration_cpu_ns": 682167.0, "calibration_gpu_ticks": 682167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 126000.0, "compute_span_ns": 126000.0, "command_buffer_ns": 125999.9698959291, "calibration_cpu_ns": 596375.0, "calibration_gpu_ticks": 596375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 134833.0, "compute_span_ns": 134833.0, "command_buffer_ns": 134833.33168551326, "calibration_cpu_ns": 606584.0, "calibration_gpu_ticks": 606584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 148250.0, "compute_span_ns": 148250.0, "command_buffer_ns": 148250.02290308475, "calibration_cpu_ns": 897875.0, "calibration_gpu_ticks": 897875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 130625.0, "compute_span_ns": 130625.0, "command_buffer_ns": 130624.97600913048, "calibration_cpu_ns": 733542.0, "calibration_gpu_ticks": 733542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 131958.0, "compute_span_ns": 131958.0, "command_buffer_ns": 131916.66221246123, "calibration_cpu_ns": 636583.0, "calibration_gpu_ticks": 636583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137625.0, "compute_span_ns": 137625.0, "command_buffer_ns": 137624.97110292315, "calibration_cpu_ns": 891500.0, "calibration_gpu_ticks": 891500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 131250.0, "compute_span_ns": 131250.0, "command_buffer_ns": 131250.0098720193, "calibration_cpu_ns": 620458.0, "calibration_gpu_ticks": 620458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 156375.0, "compute_span_ns": 156375.0, "command_buffer_ns": 156374.99745935202, "calibration_cpu_ns": 943042.0, "calibration_gpu_ticks": 943042.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 126.66475, 128.4719375, 124.375703125, 126.257796875, 128.1008125, 127.022765625, 125.800828125, 129.9928125, 125.895875 ], "compute_throughput_us_p50": 126.66475, "compute_throughput_us_p90": 128.7761125, "command_buffer_throughput_us": [ 142.70377505454235, 146.8398431825335, 141.70637859933777, 144.7115882911021, 144.96549465548014, 145.78320133296074, 144.29752718569944, 146.5345039832755, 145.8919286960736 ], "command_buffer_throughput_us_p50": 144.96549465548014, "command_buffer_throughput_us_p90": 146.5955718231271, "compute_latency_us": [ 145.125, 126.0, 134.833, 148.25, 130.625, 131.958, 137.625, 131.25, 156.375 ], "compute_latency_us_p50": 134.833, "compute_latency_us_p90": 149.875, "command_buffer_latency_us": [ 145.12502821162343, 125.9999698959291, 134.83333168551326, 148.25002290308475, 130.62497600913048, 131.91666221246123, 137.62497110292315, 131.2500098720193, 156.37499745935202 ], "command_buffer_latency_us_p50": 134.83333168551326, "command_buffer_latency_us_p90": 149.8750178143382, "counter_control_throughput_ratio": 1.1933312980254807, "counter_control_latency_ratio": 1.0199740765910716 }, "download_ms": 0.732958, "allocation_upload_ms": 16.758875, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.42199475376594603, "device_compute_slowdown": 0.42687228984385944, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.599916, "warmup_ms": 200.290208, "repetitions": 219, "throughput_us": [ 143.07058447488583, 134.08504566210047, 129.9554794520548, 132.45947488584474, 139.86948401826484, 200.77435159817352, 140.40753424657535, 133.46936529680363, 130.656200913242 ], "latency_us": [ 535.0, 439.25, 426.042, 407.58299999999997, 439.083, 757.042, 410.08299999999997, 398.54200000000003, 492.041 ], "throughput_us_p50": 134.08504566210047, "throughput_us_p90": 154.61133789954337, "latency_us_p50": 439.083, "latency_us_p90": 579.4084, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7909083.2732617855, "command_buffers": 11 }, { "command_buffer_ns": 7918625.022284687, "command_buffers": 11 }, { "command_buffer_ns": 7896833.354607224, "command_buffers": 11 }, { "command_buffer_ns": 7887249.98710677, "command_buffers": 11 }, { "command_buffer_ns": 8007916.621863842, "command_buffers": 11 }, { "command_buffer_ns": 7857833.348680288, "command_buffers": 11 }, { "command_buffer_ns": 7818750.047590584, "command_buffers": 11 }, { "command_buffer_ns": 7815208.402462304, "command_buffers": 11 }, { "command_buffer_ns": 7919916.5920726955, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 128666.63746535778, "command_buffers": 1 }, { "command_buffer_ns": 142749.98102337122, "command_buffers": 1 }, { "command_buffer_ns": 142499.97911974788, "command_buffers": 1 }, { "command_buffer_ns": 151458.37096497416, "command_buffers": 1 }, { "command_buffer_ns": 124083.36624503136, "command_buffers": 1 }, { "command_buffer_ns": 130041.67703911662, "command_buffers": 1 }, { "command_buffer_ns": 129541.67323186994, "command_buffers": 1 }, { "command_buffer_ns": 131624.98362362385, "command_buffers": 1 }, { "command_buffer_ns": 173999.98614564538, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 123.5794261447154, 123.72851597319823, 123.38802116573788, 123.23828104854329, 125.12369721662253, 122.7786460731295, 122.16796949360287, 122.1126312884735, 123.74869675113587 ], "command_buffer_throughput_us_p50": 123.38802116573788, "command_buffer_throughput_us_p90": 124.0236968442332, "command_buffer_latency_us": [ 128.66663746535778, 142.74998102337122, 142.49997911974788, 151.45837096497416, 124.08336624503136, 130.04167703911662, 129.54167323186994, 131.62498362362385, 173.99998614564538 ], "command_buffer_latency_us_p50": 131.62498362362385, "command_buffer_latency_us_p90": 155.9666940011084 }, "throughput": [ { "compute_ns": 8174786.0, "compute_span_ns": 9540875.0, "command_buffer_ns": 9317125.13556704, "calibration_cpu_ns": 10645000.0, "calibration_gpu_ticks": 10645000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8377501.0, "compute_span_ns": 9789583.0, "command_buffer_ns": 9578916.7098701, "calibration_cpu_ns": 11129708.0, "calibration_gpu_ticks": 11129708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8339668.0, "compute_span_ns": 9398625.0, "command_buffer_ns": 9243791.573680937, "calibration_cpu_ns": 10431833.0, "calibration_gpu_ticks": 10431833.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8380496.0, "compute_span_ns": 9798584.0, "command_buffer_ns": 9572041.570208967, "calibration_cpu_ns": 11032042.0, "calibration_gpu_ticks": 11032042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8243876.0, "compute_span_ns": 9574583.0, "command_buffer_ns": 9345166.676212102, "calibration_cpu_ns": 11002792.0, "calibration_gpu_ticks": 11002792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8191917.0, "compute_span_ns": 9693334.0, "command_buffer_ns": 9436791.588086635, "calibration_cpu_ns": 10720375.0, "calibration_gpu_ticks": 10720375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8222878.0, "compute_span_ns": 9691083.0, "command_buffer_ns": 9458291.63538292, "calibration_cpu_ns": 10811416.0, "calibration_gpu_ticks": 10811416.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8263500.0, "compute_span_ns": 9731917.0, "command_buffer_ns": 9487416.653428227, "calibration_cpu_ns": 10903542.0, "calibration_gpu_ticks": 10903542.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8257914.0, "compute_span_ns": 9709000.0, "command_buffer_ns": 9477416.810113937, "calibration_cpu_ns": 10865584.0, "calibration_gpu_ticks": 10865584.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 130042.0, "compute_span_ns": 130042.0, "command_buffer_ns": 130083.35372433066, "calibration_cpu_ns": 677542.0, "calibration_gpu_ticks": 677542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 154500.0, "compute_span_ns": 154500.0, "command_buffer_ns": 154500.0122860074, "calibration_cpu_ns": 952584.0, "calibration_gpu_ticks": 952584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 130333.0, "compute_span_ns": 130333.0, "command_buffer_ns": 130291.62073507905, "calibration_cpu_ns": 614833.0, "calibration_gpu_ticks": 614833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136167.0, "compute_span_ns": 136167.0, "command_buffer_ns": 136208.31305161119, "calibration_cpu_ns": 712458.0, "calibration_gpu_ticks": 712458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 133083.0, "compute_span_ns": 133083.0, "command_buffer_ns": 133083.31836014986, "calibration_cpu_ns": 680709.0, "calibration_gpu_ticks": 680709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 132792.0, "compute_span_ns": 132792.0, "command_buffer_ns": 132833.31645652652, "calibration_cpu_ns": 793375.0, "calibration_gpu_ticks": 793375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 130042.0, "compute_span_ns": 130042.0, "command_buffer_ns": 130083.35372433066, "calibration_cpu_ns": 639459.0, "calibration_gpu_ticks": 639459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135625.0, "compute_span_ns": 135625.0, "command_buffer_ns": 135624.95587393641, "calibration_cpu_ns": 769834.0, "calibration_gpu_ticks": 769834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 129958.0, "compute_span_ns": 129958.0, "command_buffer_ns": 129916.6469834745, "calibration_cpu_ns": 636209.0, "calibration_gpu_ticks": 636209.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 127.73103125, 130.898453125, 130.3073125, 130.94525, 128.8105625, 127.998703125, 128.48246875, 129.1171875, 129.02990625 ], "compute_throughput_us_p50": 129.02990625, "compute_throughput_us_p90": 130.9078125, "command_buffer_throughput_us": [ 145.580080243235, 149.6705735917203, 144.43424333876465, 149.5631495345151, 146.0182293158141, 147.44986856385367, 147.78580680285813, 148.24088520981604, 148.08463765803026 ], "command_buffer_throughput_us_p50": 147.78580680285813, "command_buffer_throughput_us_p90": 149.58463434595615, "compute_latency_us": [ 130.042, 154.5, 130.333, 136.167, 133.083, 132.792, 130.042, 135.625, 129.958 ], "compute_latency_us_p50": 132.792, "compute_latency_us_p90": 139.83360000000002, "command_buffer_latency_us": [ 130.08335372433066, 154.5000122860074, 130.29162073507905, 136.20831305161119, 133.08331836014986, 132.83331645652652, 130.08335372433066, 135.62495587393641, 129.9166469834745 ], "command_buffer_latency_us_p50": 132.83331645652652, "command_buffer_latency_us_p90": 139.86665289849043, "counter_control_throughput_ratio": 1.2009406625646408, "counter_control_latency_ratio": 1.0110107506256572 }, "download_ms": 0.573208, "allocation_upload_ms": 2.187875, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-200xztc0/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "8bb581805080afef9f47b3147742261e73f0bbc7f1d60afd69631b609a5e3f17", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.562458, "capture_ms": 0.061917, "compile_ms": 51.379958, "allocation_upload_ms": 5.428833, "cold_call_ms": 6.403875, "warmup_ms": 200.498416, "download_ms": 3.385083, "repetitions": 494, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 51.933451417, 52.0436072874, 52.6306497976, 52.145917004, 52.3895910931, 52.0655364372, 53.3641194332, 52.1474352227, 51.8901821862 ], "latency_us": [ 409.916, 360, 384.834, 313, 298.542, 320.75, 297.208, 303.291, 340.666 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3334292, "compute_span_ns": 3334292, "calibration_cpu_ns": 3874667, "calibration_gpu_ticks": 3874667, "compute_passes": 1, "command_buffer_ns": 3334374.98193, "command_buffers": 2 }, { "compute_ns": 3325875, "compute_span_ns": 3325875, "calibration_cpu_ns": 4365375, "calibration_gpu_ticks": 4365375, "compute_passes": 1, "command_buffer_ns": 3325958.32879, "command_buffers": 2 }, { "compute_ns": 3313708, "compute_span_ns": 3313708, "calibration_cpu_ns": 4114166, "calibration_gpu_ticks": 4114166, "compute_passes": 1, "command_buffer_ns": 3313791.64709, "command_buffers": 2 }, { "compute_ns": 3344792, "compute_span_ns": 3344792, "calibration_cpu_ns": 4081250, "calibration_gpu_ticks": 4081250, "compute_passes": 1, "command_buffer_ns": 3344916.68036, "command_buffers": 2 }, { "compute_ns": 3188292, "compute_span_ns": 3188292, "calibration_cpu_ns": 3746500, "calibration_gpu_ticks": 3746500, "compute_passes": 1, "command_buffer_ns": 3188333.35768, "command_buffers": 2 }, { "compute_ns": 3305083, "compute_span_ns": 3305083, "calibration_cpu_ns": 4077792, "calibration_gpu_ticks": 4077792, "compute_passes": 1, "command_buffer_ns": 3305083.31535, "command_buffers": 2 }, { "compute_ns": 3254709, "compute_span_ns": 3254709, "calibration_cpu_ns": 4013042, "calibration_gpu_ticks": 4013042, "compute_passes": 1, "command_buffer_ns": 3254791.66349, "command_buffers": 2 }, { "compute_ns": 3318333, "compute_span_ns": 3318333, "calibration_cpu_ns": 3940541, "calibration_gpu_ticks": 3940541, "compute_passes": 1, "command_buffer_ns": 3318374.97652, "command_buffers": 2 }, { "compute_ns": 3322458, "compute_span_ns": 3322458, "calibration_cpu_ns": 4076333, "calibration_gpu_ticks": 4076333, "compute_passes": 1, "command_buffer_ns": 3322500.03703, "command_buffers": 2 } ], "latency": [ { "compute_ns": 65416, "compute_span_ns": 65416, "calibration_cpu_ns": 547500, "calibration_gpu_ticks": 547500, "compute_passes": 1, "command_buffer_ns": 65458.3564028, "command_buffers": 2 }, { "compute_ns": 75625, "compute_span_ns": 75625, "calibration_cpu_ns": 746750, "calibration_gpu_ticks": 746750, "compute_passes": 1, "command_buffer_ns": 75666.6995585, "command_buffers": 2 }, { "compute_ns": 98334, "compute_span_ns": 98334, "calibration_cpu_ns": 811750, "calibration_gpu_ticks": 811750, "compute_passes": 1, "command_buffer_ns": 98375.02148, "command_buffers": 2 }, { "compute_ns": 65125, "compute_span_ns": 65125, "calibration_cpu_ns": 520042, "calibration_gpu_ticks": 520042, "compute_passes": 1, "command_buffer_ns": 65125.0011288, "command_buffers": 2 }, { "compute_ns": 63333, "compute_span_ns": 63333, "calibration_cpu_ns": 559833, "calibration_gpu_ticks": 559833, "compute_passes": 1, "command_buffer_ns": 63291.634433, "command_buffers": 2 }, { "compute_ns": 107417, "compute_span_ns": 107417, "calibration_cpu_ns": 681584, "calibration_gpu_ticks": 681584, "compute_passes": 1, "command_buffer_ns": 107500.003651, "command_buffers": 2 }, { "compute_ns": 82709, "compute_span_ns": 82709, "calibration_cpu_ns": 783084, "calibration_gpu_ticks": 783084, "compute_passes": 1, "command_buffer_ns": 82791.7247079, "command_buffers": 2 }, { "compute_ns": 111250, "compute_span_ns": 111250, "calibration_cpu_ns": 1539417, "calibration_gpu_ticks": 1539417, "compute_passes": 1, "command_buffer_ns": 111291.650683, "command_buffers": 2 }, { "compute_ns": 100959, "compute_span_ns": 100959, "calibration_cpu_ns": 1232125, "calibration_gpu_ticks": 1232125, "compute_passes": 1, "command_buffer_ns": 101000.012364, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3503000.01446, "command_buffers": 2 }, { "command_buffer_ns": 3400916.64111, "command_buffers": 2 }, { "command_buffer_ns": 3334708.279, "command_buffers": 2 }, { "command_buffer_ns": 3296166.6584, "command_buffers": 2 }, { "command_buffer_ns": 3316666.69808, "command_buffers": 2 }, { "command_buffer_ns": 3339916.64229, "command_buffers": 2 }, { "command_buffer_ns": 3281124.98391, "command_buffers": 2 }, { "command_buffer_ns": 3214208.35098, "command_buffers": 2 }, { "command_buffer_ns": 3247250.04612, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 103833.328467, "command_buffers": 2 }, { "command_buffer_ns": 64458.2905807, "command_buffers": 2 }, { "command_buffer_ns": 72000.0243746, "command_buffers": 2 }, { "command_buffer_ns": 66666.6892357, "command_buffers": 2 }, { "command_buffer_ns": 77416.6546762, "command_buffers": 2 }, { "command_buffer_ns": 93291.6300371, "command_buffers": 2 }, { "command_buffer_ns": 82999.9917187, "command_buffers": 2 }, { "command_buffer_ns": 90416.695457, "command_buffers": 2 }, { "command_buffer_ns": 70250.0110492, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 54.734375225937505, 53.13932251734375, 52.104816859375, 51.5026040375, 51.8229171575, 52.18619753578125, 51.26757787359375, 50.2220054840625, 50.738281970625 ], "command_buffer_throughput_us_p50": 51.8229171575, "command_buffer_throughput_us_p90": 53.458333059062504, "command_buffer_latency_us": [ 103.833328467, 64.4582905807, 72.0000243746, 66.6666892357, 77.41665467620001, 93.29163003709999, 82.9999917187, 90.41669545699999, 70.2500110492 ], "command_buffer_latency_us_p50": 77.41665467620001, "command_buffer_latency_us_p90": 95.39996972307999 }, "compute_throughput_us": [ 52.0983125, 51.966796875, 51.7766875, 52.262375, 49.8170625, 51.641921875, 50.854828125, 51.848953125, 51.91340625 ], "compute_throughput_us_p50": 51.848953125, "compute_throughput_us_p90": 52.131125, "command_buffer_throughput_us": [ 52.099609092656245, 51.96809888734375, 51.77799448578125, 52.264323130625, 49.81770871375, 51.64192680234375, 50.856119742031254, 51.849609008125, 51.91406307859375 ], "command_buffer_throughput_us_p50": 51.849609008125, "command_buffer_throughput_us_p90": 52.13255190024999, "compute_latency_us": [ 65.416, 75.625, 98.334, 65.125, 63.333, 107.417, 82.709, 111.25, 100.959 ], "compute_latency_us_p50": 82.709, "compute_latency_us_p90": 108.1836, "command_buffer_latency_us": [ 65.45835640279999, 75.6666995585, 98.37502148, 65.12500112880001, 63.291634433, 107.500003651, 82.79172470789999, 111.291650683, 101.000012364 ], "command_buffer_latency_us_p50": 82.79172470789999, "command_buffer_latency_us_p90": 108.2583330574, "counter_control_throughput_ratio": 0.9919743013298387, "counter_control_latency_ratio": 1.1523006255571873 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 52.145917004, "throughput_us_p90": 52.77734372472, "latency_us_p50": 320.75, "latency_us_p90": 389.8504 }, "slowdown": 0.38890181038838395, "device_compute_slowdown": 0.40183671082067457, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-ccv1qgpr/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "54d99166e42515c7bfff1cac173ad46750fa91f9c92e668c07d69318fdc050bf", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.539916, "capture_ms": 0.060959, "compile_ms": 58.582041, "allocation_upload_ms": 1.73675, "cold_call_ms": 2.27025, "warmup_ms": 200.263667, "download_ms": 0.367459, "repetitions": 7067, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.12976991651, 4.1687126079, 4.12219937739, 4.16416669025, 4.26096066223, 4.15354223857, 4.20431229659, 4.30201415028, 4.16065289373 ], "latency_us": [ 286.166, 201.083, 189.542, 171.75, 176, 202.041, 187.334, 312.916, 202.5 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 56, "normalized_kernel_cost": 56, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 260541, "compute_span_ns": 260541, "calibration_cpu_ns": 915542, "calibration_gpu_ticks": 915542, "compute_passes": 1, "command_buffer_ns": 260583.357885, "command_buffers": 2 }, { "compute_ns": 245167, "compute_span_ns": 245167, "calibration_cpu_ns": 891875, "calibration_gpu_ticks": 891875, "compute_passes": 1, "command_buffer_ns": 245250.004809, "command_buffers": 2 }, { "compute_ns": 254875, "compute_span_ns": 254875, "calibration_cpu_ns": 685708, "calibration_gpu_ticks": 685708, "compute_passes": 1, "command_buffer_ns": 255000.020843, "command_buffers": 2 }, { "compute_ns": 260750, "compute_span_ns": 260750, "calibration_cpu_ns": 896500, "calibration_gpu_ticks": 896500, "compute_passes": 1, "command_buffer_ns": 260833.359789, "command_buffers": 2 }, { "compute_ns": 244709, "compute_span_ns": 244709, "calibration_cpu_ns": 637458, "calibration_gpu_ticks": 637458, "compute_passes": 1, "command_buffer_ns": 244833.354373, "command_buffers": 2 }, { "compute_ns": 245000, "compute_span_ns": 245000, "calibration_cpu_ns": 676709, "calibration_gpu_ticks": 676709, "compute_passes": 1, "command_buffer_ns": 245041.679591, "command_buffers": 2 }, { "compute_ns": 244250, "compute_span_ns": 244250, "calibration_cpu_ns": 806542, "calibration_gpu_ticks": 806542, "compute_passes": 1, "command_buffer_ns": 244249.997195, "command_buffers": 2 }, { "compute_ns": 244958, "compute_span_ns": 244958, "calibration_cpu_ns": 804083, "calibration_gpu_ticks": 804083, "compute_passes": 1, "command_buffer_ns": 244958.326221, "command_buffers": 2 }, { "compute_ns": 259084, "compute_span_ns": 259084, "calibration_cpu_ns": 875875, "calibration_gpu_ticks": 875875, "compute_passes": 1, "command_buffer_ns": 259124.964941, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6333, "compute_span_ns": 6333, "calibration_cpu_ns": 449292, "calibration_gpu_ticks": 449292, "compute_passes": 1, "command_buffer_ns": 6291.66606814, "command_buffers": 2 }, { "compute_ns": 6959, "compute_span_ns": 6959, "calibration_cpu_ns": 468459, "calibration_gpu_ticks": 468459, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "compute_ns": 6833, "compute_span_ns": 6833, "calibration_cpu_ns": 421084, "calibration_gpu_ticks": 421084, "compute_passes": 1, "command_buffer_ns": 6875.02324581, "command_buffers": 2 }, { "compute_ns": 6834, "compute_span_ns": 6834, "calibration_cpu_ns": 434959, "calibration_gpu_ticks": 434959, "compute_passes": 1, "command_buffer_ns": 6916.64172336, "command_buffers": 2 }, { "compute_ns": 8209, "compute_span_ns": 8209, "calibration_cpu_ns": 676208, "calibration_gpu_ticks": 676208, "compute_passes": 1, "command_buffer_ns": 8333.35798234, "command_buffers": 2 }, { "compute_ns": 7334, "compute_span_ns": 7334, "calibration_cpu_ns": 805958, "calibration_gpu_ticks": 805958, "compute_passes": 1, "command_buffer_ns": 7416.64553061, "command_buffers": 2 }, { "compute_ns": 6916, "compute_span_ns": 6916, "calibration_cpu_ns": 574917, "calibration_gpu_ticks": 574917, "compute_passes": 1, "command_buffer_ns": 6875.02324581, "command_buffers": 2 }, { "compute_ns": 6875, "compute_span_ns": 6875, "calibration_cpu_ns": 458542, "calibration_gpu_ticks": 458542, "compute_passes": 1, "command_buffer_ns": 6916.69993103, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 511125, "calibration_gpu_ticks": 511125, "compute_passes": 1, "command_buffer_ns": 7375.02705306, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 263875.00111, "command_buffers": 2 }, { "command_buffer_ns": 252125.028055, "command_buffers": 2 }, { "command_buffer_ns": 244125.025347, "command_buffers": 2 }, { "command_buffer_ns": 245083.356276, "command_buffers": 2 }, { "command_buffer_ns": 244666.647632, "command_buffers": 2 }, { "command_buffer_ns": 260791.624896, "command_buffers": 2 }, { "command_buffer_ns": 260166.707449, "command_buffers": 2 }, { "command_buffer_ns": 246958.283242, "command_buffers": 2 }, { "command_buffer_ns": 261958.339252, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6249.98938292, "command_buffers": 2 }, { "command_buffer_ns": 8791.68510437, "command_buffers": 2 }, { "command_buffer_ns": 6958.37661624, "command_buffers": 2 }, { "command_buffer_ns": 7375.02705306, "command_buffers": 2 }, { "command_buffer_ns": 6666.6980274, "command_buffers": 2 }, { "command_buffer_ns": 6875.02324581, "command_buffers": 2 }, { "command_buffer_ns": 6875.02324581, "command_buffers": 2 }, { "command_buffer_ns": 8124.97455627, "command_buffers": 2 }, { "command_buffer_ns": 7250.05520508, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.12304689234375, 3.939453563359375, 3.814453521046875, 3.8294274418125003, 3.82291636925, 4.074869139, 4.065104803890625, 3.85872317565625, 4.0930990508125005 ], "command_buffer_throughput_us_p50": 3.939453563359375, "command_buffer_throughput_us_p90": 4.09908861911875, "command_buffer_latency_us": [ 6.24998938292, 8.79168510437, 6.95837661624, 7.37502705306, 6.6666980274, 6.87502324581, 6.87502324581, 8.12497455627, 7.25005520508 ], "command_buffer_latency_us_p50": 6.95837661624, "command_buffer_latency_us_p90": 8.25831666589 }, "compute_throughput_us": [ 4.070953125, 3.830734375, 3.982421875, 4.07421875, 3.823578125, 3.828125, 3.81640625, 3.82746875, 4.0481875 ], "compute_throughput_us_p50": 3.830734375, "compute_throughput_us_p90": 4.07160625, "command_buffer_throughput_us": [ 4.071614966953125, 3.8320313251406253, 3.9843753256718752, 4.075521246703125, 3.8255211620781253, 3.828776243609375, 3.816406206171875, 3.827473847203125, 4.0488275772031255 ], "command_buffer_throughput_us_p50": 3.8320313251406253, "command_buffer_throughput_us_p90": 4.072396222903125, "compute_latency_us": [ 6.333, 6.959, 6.833, 6.834, 8.209, 7.334, 6.916, 6.875, 7.25 ], "compute_latency_us_p50": 6.916, "compute_latency_us_p90": 7.5089999999999995, "command_buffer_latency_us": [ 6.29166606814, 7.0416717790100005, 6.87502324581, 6.91664172336, 8.333357982339999, 7.416645530609999, 6.87502324581, 6.91669993103, 7.37502705306 ], "command_buffer_latency_us_p50": 6.91669993103, "command_buffer_latency_us_p90": 7.599988020955999, "counter_control_throughput_ratio": 0.9891838743554014, "counter_control_latency_ratio": 1.0 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.16416669025, "throughput_us_p90": 4.26917135984, "latency_us_p50": 201.083, "latency_us_p90": 291.516 }, "torch": { "cold_call_ms": 3.180167, "warmup_ms": 200.23225, "repetitions": 3328, "throughput_us": [ 7.661157752403846, 7.639060096153846, 7.570074519230769, 7.614107572115385, 7.713429086538462, 7.519831730769231, 7.487067007211538, 7.510091045673078, 7.546850060096154 ], "latency_us": [ 413.08299999999997, 275.959, 277.041, 282.292, 249.417, 236.667, 305.25, 234.16600000000003, 227.167 ], "throughput_us_p50": 7.570074519230769, "throughput_us_p90": 7.6716120192307695, "latency_us_p50": 275.959, "latency_us_p90": 326.8166, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 306458.32885056734, "command_buffers": 1 }, { "command_buffer_ns": 342125.0148676336, "command_buffers": 1 }, { "command_buffer_ns": 289875.0244639814, "command_buffers": 1 }, { "command_buffer_ns": 304999.9941140413, "command_buffers": 1 }, { "command_buffer_ns": 321124.97137859464, "command_buffers": 1 }, { "command_buffer_ns": 304624.96215477586, "command_buffers": 1 }, { "command_buffer_ns": 305125.02416968346, "command_buffers": 1 }, { "command_buffer_ns": 305708.3231396973, "command_buffers": 1 }, { "command_buffer_ns": 307874.9869018793, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 7666.705641895533, "command_buffers": 1 }, { "command_buffer_ns": 8333.299774676561, "command_buffers": 1 }, { "command_buffer_ns": 8833.36178958416, "command_buffers": 1 }, { "command_buffer_ns": 8166.651241481304, "command_buffers": 1 }, { "command_buffer_ns": 9375.042282044888, "command_buffers": 1 }, { "command_buffer_ns": 8041.67939350009, "command_buffers": 1 }, { "command_buffer_ns": 9750.016033649445, "command_buffers": 1 }, { "command_buffer_ns": 8583.359885960817, "command_buffers": 1 }, { "command_buffer_ns": 8333.357982337475, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 4.788411388290115, 5.345703357306775, 4.529297257249709, 4.765624908031896, 5.017577677790541, 4.759765033668373, 4.767578502651304, 4.7766925490577705, 4.810546670341864 ], "command_buffer_throughput_us_p50": 4.7766925490577705, "command_buffer_throughput_us_p90": 5.083202813693788, "command_buffer_latency_us": [ 7.666705641895533, 8.333299774676561, 8.83336178958416, 8.166651241481304, 9.375042282044888, 8.04167939350009, 9.750016033649445, 8.583359885960817, 8.333357982337475 ], "command_buffer_latency_us_p50": 8.333357982337475, "command_buffer_latency_us_p90": 9.450037032365799 }, "throughput": [ { "compute_ns": 289750.0, "compute_span_ns": 289750.0, "command_buffer_ns": 289749.99440833926, "calibration_cpu_ns": 972250.0, "calibration_gpu_ticks": 972250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288375.0, "compute_span_ns": 288375.0, "command_buffer_ns": 288375.01304224133, "calibration_cpu_ns": 928334.0, "calibration_gpu_ticks": 928334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288916.0, "compute_span_ns": 288916.0, "command_buffer_ns": 288874.9586418271, "calibration_cpu_ns": 909125.0, "calibration_gpu_ticks": 909125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 304875.0, "compute_span_ns": 304875.0, "command_buffer_ns": 304875.0222660601, "calibration_cpu_ns": 939583.0, "calibration_gpu_ticks": 939583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 308916.0, "compute_span_ns": 308916.0, "command_buffer_ns": 308875.0527240336, "calibration_cpu_ns": 978000.0, "calibration_gpu_ticks": 978000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289583.0, "compute_span_ns": 289583.0, "command_buffer_ns": 289583.345875144, "calibration_cpu_ns": 973167.0, "calibration_gpu_ticks": 973167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289792.0, "compute_span_ns": 289792.0, "command_buffer_ns": 289833.28957110643, "calibration_cpu_ns": 907292.0, "calibration_gpu_ticks": 907292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 290375.0, "compute_span_ns": 290375.0, "command_buffer_ns": 290374.97006356716, "calibration_cpu_ns": 962667.0, "calibration_gpu_ticks": 962667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 314500.0, "compute_span_ns": 314500.0, "command_buffer_ns": 314500.00824406743, "calibration_cpu_ns": 927459.0, "calibration_gpu_ticks": 927459.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 7791.0, "compute_span_ns": 7791.0, "command_buffer_ns": 7750.0008046627045, "calibration_cpu_ns": 589791.0, "calibration_gpu_ticks": 589791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9166.0, "compute_span_ns": 9166.0, "command_buffer_ns": 9124.982170760632, "calibration_cpu_ns": 697833.0, "calibration_gpu_ticks": 697833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8167.0, "compute_span_ns": 8167.0, "command_buffer_ns": 8166.651241481304, "calibration_cpu_ns": 485500.0, "calibration_gpu_ticks": 485500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8208.0, "compute_span_ns": 8208.0, "command_buffer_ns": 8166.651241481304, "calibration_cpu_ns": 476709.0, "calibration_gpu_ticks": 476709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8500.0, "compute_span_ns": 8500.0, "command_buffer_ns": 8499.948307871819, "calibration_cpu_ns": 715667.0, "calibration_gpu_ticks": 715667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8042.0, "compute_span_ns": 8042.0, "command_buffer_ns": 8041.67939350009, "calibration_cpu_ns": 449167.0, "calibration_gpu_ticks": 449167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9833.0, "compute_span_ns": 9833.0, "command_buffer_ns": 9791.692718863487, "calibration_cpu_ns": 556750.0, "calibration_gpu_ticks": 556750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 9292.0, "compute_span_ns": 9292.0, "command_buffer_ns": 9333.365596830845, "calibration_cpu_ns": 612125.0, "calibration_gpu_ticks": 612125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8167.0, "compute_span_ns": 8167.0, "command_buffer_ns": 8208.38613435626, "calibration_cpu_ns": 466500.0, "calibration_gpu_ticks": 466500.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 4.52734375, 4.505859375, 4.5143125, 4.763671875, 4.8268125, 4.524734375, 4.528, 4.537109375, 4.9140625 ], "compute_throughput_us_p50": 4.528, "compute_throughput_us_p90": 4.8442625, "command_buffer_throughput_us": [ 4.527343662630301, 4.505859578785021, 4.5136712287785485, 4.763672222907189, 4.826172698813025, 4.524739779299125, 4.528645149548538, 4.537108907243237, 4.914062628813554 ], "command_buffer_throughput_us_p50": 4.528645149548538, "command_buffer_throughput_us_p90": 4.843750684813131, "compute_latency_us": [ 7.791, 9.166, 8.167, 8.208, 8.5, 8.042, 9.833, 9.292, 8.167 ], "compute_latency_us_p50": 8.208, "compute_latency_us_p90": 9.4002, "command_buffer_latency_us": [ 7.7500008046627045, 9.124982170760632, 8.166651241481304, 8.166651241481304, 8.499948307871819, 8.04167939350009, 9.791692718863487, 9.333365596830845, 8.20838613435626 ], "command_buffer_latency_us_p50": 8.20838613435626, "command_buffer_latency_us_p90": 9.425031021237373, "counter_control_throughput_ratio": 0.9506225091560638, "counter_control_latency_ratio": 1.0 }, "download_ms": 0.27125, "allocation_upload_ms": 0.860792, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.5500826550216233, "device_compute_slowdown": 0.846010241828622, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.280959, "warmup_ms": 200.029583, "repetitions": 2984, "throughput_us": [ 8.007735589812333, 7.913915884718499, 7.841711461126006, 8.238717493297587, 8.070277815013405, 8.267342493297587, 8.265792560321715, 8.465678284182305, 7.812806970509384 ], "latency_us": [ 393.125, 259.25, 210.916, 192.125, 988.458, 228.542, 191.625, 202.459, 194.542 ], "throughput_us_p50": 8.070277815013405, "throughput_us_p90": 8.30700965147453, "latency_us_p50": 210.916, "latency_us_p90": 512.1916000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 374583.3528228104, "command_buffers": 1 }, { "command_buffer_ns": 366708.3219625056, "command_buffers": 1 }, { "command_buffer_ns": 341750.041116029, "command_buffers": 1 }, { "command_buffer_ns": 289583.345875144, "command_buffers": 1 }, { "command_buffer_ns": 365124.95717033744, "command_buffers": 1 }, { "command_buffer_ns": 289541.66918992996, "command_buffers": 1 }, { "command_buffer_ns": 328666.64696484804, "command_buffers": 1 }, { "command_buffer_ns": 336624.97298792005, "command_buffers": 1 }, { "command_buffer_ns": 311249.983496964, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 9500.014130026102, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 8041.67939350009, "command_buffers": 1 }, { "command_buffer_ns": 8333.299774676561, "command_buffers": 1 }, { "command_buffer_ns": 8999.952115118504, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 5.852864887856413, 5.72981753066415, 5.3398443924379535, 4.524739779299125, 5.7050774557865225, 4.524088581092656, 5.135416358825751, 5.259765202936251, 4.863280992140062 ], "command_buffer_throughput_us_p50": 5.259765202936251, "command_buffer_throughput_us_p90": 5.754427002102602, "command_buffer_latency_us": [ 8.124974556267262, 8.000002708286047, 8.000002708286047, 9.500014130026102, 8.000002708286047, 8.04167939350009, 8.333299774676561, 8.999952115118504, 8.124974556267262 ], "command_buffer_latency_us_p50": 8.124974556267262, "command_buffer_latency_us_p90": 9.099964518100023 }, "throughput": [ { "compute_ns": 343625.0, "compute_span_ns": 343625.0, "command_buffer_ns": 343625.02628937364, "calibration_cpu_ns": 1009166.0, "calibration_gpu_ticks": 1009166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 289958.0, "compute_span_ns": 289958.0, "command_buffer_ns": 289916.70114919543, "calibration_cpu_ns": 863292.0, "calibration_gpu_ticks": 863292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 361041.0, "compute_span_ns": 361041.0, "command_buffer_ns": 360999.95486438274, "calibration_cpu_ns": 970333.0, "calibration_gpu_ticks": 970333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 338417.0, "compute_span_ns": 338417.0, "command_buffer_ns": 338416.6629984975, "calibration_cpu_ns": 932792.0, "calibration_gpu_ticks": 932792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 362084.0, "compute_span_ns": 362084.0, "command_buffer_ns": 362124.99253451824, "calibration_cpu_ns": 1244792.0, "calibration_gpu_ticks": 1244792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 339542.0, "compute_span_ns": 339542.0, "command_buffer_ns": 339583.3191461861, "calibration_cpu_ns": 972417.0, "calibration_gpu_ticks": 972417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 350625.0, "compute_span_ns": 350625.0, "command_buffer_ns": 350625.0213831663, "calibration_cpu_ns": 1334916.0, "calibration_gpu_ticks": 1334916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 337000.0, "compute_span_ns": 337000.0, "command_buffer_ns": 337000.0049471855, "calibration_cpu_ns": 1047958.0, "calibration_gpu_ticks": 1047958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 363250.0, "compute_span_ns": 363250.0, "command_buffer_ns": 363250.03020465374, "calibration_cpu_ns": 1116708.0, "calibration_gpu_ticks": 1116708.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 8333.0, "compute_span_ns": 8333.0, "command_buffer_ns": 8291.681297123432, "calibration_cpu_ns": 434500.0, "calibration_gpu_ticks": 434500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8000.0, "compute_span_ns": 8000.0, "command_buffer_ns": 8000.002708286047, "calibration_cpu_ns": 443792.0, "calibration_gpu_ticks": 443792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8000.0, "compute_span_ns": 8000.0, "command_buffer_ns": 8000.002708286047, "calibration_cpu_ns": 463292.0, "calibration_gpu_ticks": 463292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8042.0, "compute_span_ns": 8042.0, "command_buffer_ns": 8041.621185839176, "calibration_cpu_ns": 418958.0, "calibration_gpu_ticks": 418958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8542.0, "compute_span_ns": 8542.0, "command_buffer_ns": 8583.301678299904, "calibration_cpu_ns": 657459.0, "calibration_gpu_ticks": 657459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8125.032763928175, "calibration_cpu_ns": 444959.0, "calibration_gpu_ticks": 444959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8083.0, "compute_span_ns": 8083.0, "command_buffer_ns": 8083.356078714132, "calibration_cpu_ns": 480292.0, "calibration_gpu_ticks": 480292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8000.0, "compute_span_ns": 8000.0, "command_buffer_ns": 8000.002708286047, "calibration_cpu_ns": 435417.0, "calibration_gpu_ticks": 435417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8042.0, "compute_span_ns": 8042.0, "command_buffer_ns": 8083.356078714132, "calibration_cpu_ns": 437583.0, "calibration_gpu_ticks": 437583.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 5.369140625, 4.53059375, 5.641265625, 5.287765625, 5.6575625, 5.30534375, 5.478515625, 5.265625, 5.67578125 ], "compute_throughput_us_p50": 5.369140625, "compute_throughput_us_p90": 5.66120625, "command_buffer_throughput_us": [ 5.369141035771463, 4.529948455456179, 5.64062429475598, 5.287760359351523, 5.6582030083518475, 5.305989361659158, 5.478515959111974, 5.265625077299774, 5.675781721947715 ], "command_buffer_throughput_us_p50": 5.369141035771463, "command_buffer_throughput_us_p90": 5.661718751071021, "compute_latency_us": [ 8.333, 8.0, 8.0, 8.042, 8.542, 8.125, 8.083, 8.0, 8.042 ], "compute_latency_us_p50": 8.042, "compute_latency_us_p90": 8.3748, "command_buffer_latency_us": [ 8.291681297123432, 8.000002708286047, 8.000002708286047, 8.041621185839176, 8.583301678299904, 8.125032763928175, 8.083356078714132, 8.000002708286047, 8.083356078714132 ], "command_buffer_latency_us_p50": 8.083356078714132, "command_buffer_latency_us_p90": 8.350005373358727, "counter_control_throughput_ratio": 1.0563274657860775, "counter_control_latency_ratio": 1.0 }, "download_ms": 0.475875, "allocation_upload_ms": 1.829792, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-o6cz5rxq/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "e81310ef6c7b3435af776cabf332187d02f0ddc801b9234c32af8a56116f248b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 28.9585, "capture_ms": 0.065792, "compile_ms": 65.94525, "allocation_upload_ms": 1.700709, "cold_call_ms": 2.088375, "warmup_ms": 200.137834, "download_ms": 0.291709, "repetitions": 6029, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.44813252612, 4.52011112954, 4.77045662631, 4.51434732128, 4.53763061868, 4.48615724001, 4.40817299718, 4.46916984575, 4.48080792835 ], "latency_us": [ 625.041, 244.417, 213.833, 208.75, 207.833, 236.208, 391.333, 255.584, 200.75 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 48, "normalized_kernel_cost": 48, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 300083, "compute_span_ns": 300083, "calibration_cpu_ns": 1008500, "calibration_gpu_ticks": 1008500, "compute_passes": 1, "command_buffer_ns": 300124.986097, "command_buffers": 2 }, { "compute_ns": 265333, "compute_span_ns": 265333, "calibration_cpu_ns": 728875, "calibration_gpu_ticks": 728875, "compute_passes": 1, "command_buffer_ns": 265416.689217, "command_buffers": 2 }, { "compute_ns": 292500, "compute_span_ns": 292500, "calibration_cpu_ns": 1022541, "calibration_gpu_ticks": 1022541, "compute_passes": 1, "command_buffer_ns": 292541.633826, "command_buffers": 2 }, { "compute_ns": 292875, "compute_span_ns": 292875, "calibration_cpu_ns": 1062292, "calibration_gpu_ticks": 1062292, "compute_passes": 1, "command_buffer_ns": 292958.34247, "command_buffers": 2 }, { "compute_ns": 293375, "compute_span_ns": 293375, "calibration_cpu_ns": 1280208, "calibration_gpu_ticks": 1280208, "compute_passes": 1, "command_buffer_ns": 293416.669592, "command_buffers": 2 }, { "compute_ns": 290458, "compute_span_ns": 290458, "calibration_cpu_ns": 755458, "calibration_gpu_ticks": 755458, "compute_passes": 1, "command_buffer_ns": 290500.058327, "command_buffers": 2 }, { "compute_ns": 295042, "compute_span_ns": 295042, "calibration_cpu_ns": 982875, "calibration_gpu_ticks": 982875, "compute_passes": 1, "command_buffer_ns": 295083.329547, "command_buffers": 2 }, { "compute_ns": 291750, "compute_span_ns": 291750, "calibration_cpu_ns": 1171209, "calibration_gpu_ticks": 1171209, "compute_passes": 1, "command_buffer_ns": 291750.009637, "command_buffers": 2 }, { "compute_ns": 294583, "compute_span_ns": 294583, "calibration_cpu_ns": 1293125, "calibration_gpu_ticks": 1293125, "compute_passes": 1, "command_buffer_ns": 294583.32574, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7041, "compute_span_ns": 7041, "calibration_cpu_ns": 457416, "calibration_gpu_ticks": 457416, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "compute_ns": 7166, "compute_span_ns": 7166, "calibration_cpu_ns": 452333, "calibration_gpu_ticks": 452333, "compute_passes": 1, "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "compute_ns": 7583, "compute_span_ns": 7583, "calibration_cpu_ns": 754834, "calibration_gpu_ticks": 754834, "compute_passes": 1, "command_buffer_ns": 7625.02895668, "command_buffers": 2 }, { "compute_ns": 7166, "compute_span_ns": 7166, "calibration_cpu_ns": 430416, "calibration_gpu_ticks": 430416, "compute_passes": 1, "command_buffer_ns": 7208.37851986, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 472000, "calibration_gpu_ticks": 472000, "compute_passes": 1, "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "compute_ns": 7083, "compute_span_ns": 7083, "calibration_cpu_ns": 406708, "calibration_gpu_ticks": 406708, "compute_passes": 1, "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "compute_ns": 7292, "compute_span_ns": 7292, "calibration_cpu_ns": 718458, "calibration_gpu_ticks": 718458, "compute_passes": 1, "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 433709, "calibration_gpu_ticks": 433709, "compute_passes": 1, "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "compute_ns": 7083, "compute_span_ns": 7083, "calibration_cpu_ns": 389875, "calibration_gpu_ticks": 389875, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 295333.331451, "command_buffers": 2 }, { "command_buffer_ns": 265208.363999, "command_buffers": 2 }, { "command_buffer_ns": 292583.310511, "command_buffers": 2 }, { "command_buffer_ns": 266083.341558, "command_buffers": 2 }, { "command_buffer_ns": 290750.002023, "command_buffers": 2 }, { "command_buffer_ns": 291624.979582, "command_buffers": 2 }, { "command_buffer_ns": 291500.007734, "command_buffers": 2 }, { "command_buffer_ns": 291916.658171, "command_buffers": 2 }, { "command_buffer_ns": 265666.691121, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6958.37661624, "command_buffers": 2 }, { "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "command_buffer_ns": 8791.62689671, "command_buffers": 2 }, { "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "command_buffer_ns": 6958.37661624, "command_buffers": 2 }, { "command_buffer_ns": 8249.94640425, "command_buffers": 2 }, { "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "command_buffer_ns": 7208.26210454, "command_buffers": 2 }, { "command_buffer_ns": 7124.96694177, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.614583303921875, 4.1438806874843745, 4.571614226734375, 4.1575522118437505, 4.542968781609375, 4.55664030596875, 4.55468762084375, 4.561197783921875, 4.151042048765625 ], "command_buffer_throughput_us_p50": 4.55468762084375, "command_buffer_throughput_us_p90": 4.580208042171875, "command_buffer_latency_us": [ 6.95837661624, 7.208320312200001, 8.791626896710001, 7.1249669417699995, 6.95837661624, 8.24994640425, 7.12502514943, 7.20826210454, 7.1249669417699995 ], "command_buffer_latency_us_p50": 7.12502514943, "command_buffer_latency_us_p90": 8.358282502742 }, "compute_throughput_us": [ 4.688796875, 4.145828125, 4.5703125, 4.576171875, 4.583984375, 4.53840625, 4.61003125, 4.55859375, 4.602859375 ], "compute_throughput_us_p50": 4.576171875, "compute_throughput_us_p90": 4.625784374999999, "command_buffer_throughput_us": [ 4.6894529077656255, 4.147135769015625, 4.570963028531249, 4.57747410109375, 4.584635462375, 4.5390634113593755, 4.610677024171875, 4.558593900578125, 4.6028644646875 ], "command_buffer_throughput_us_p50": 4.57747410109375, "command_buffer_throughput_us_p90": 4.626432200890625, "compute_latency_us": [ 7.041, 7.166, 7.583, 7.166, 7.125, 7.083, 7.292, 7.125, 7.083 ], "compute_latency_us_p50": 7.125, "compute_latency_us_p90": 7.3502, "command_buffer_latency_us": [ 7.0416717790100005, 7.12502514943, 7.6250289566800005, 7.20837851986, 7.1249669417699995, 7.1249669417699995, 7.45832221583, 7.1667018346499995, 7.0416717790100005 ], "command_buffer_latency_us_p50": 7.12502514943, "command_buffer_latency_us_p90": 7.491663564, "counter_control_throughput_ratio": 1.0091716854701487, "counter_control_latency_ratio": 0.9942343564527399 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.48615724001, "throughput_us_p90": 4.584195820205999, "latency_us_p50": 236.208, "latency_us_p90": 438.0746000000001 }, "slowdown": 0.5558863452834614, "device_compute_slowdown": 0.8523099308839578, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-5u5hceat/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "a8c235e7c8be57d97298e1f9f70525f6c2c7f4ae4cee9647b15503a46aab5ade", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.249333, "capture_ms": 0.060042, "compile_ms": 57.388917, "allocation_upload_ms": 2.102791, "cold_call_ms": 6.208125, "warmup_ms": 200.039875, "download_ms": 0.963708, "repetitions": 1801, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 16.2803764575, 16.0008789561, 16.1503098279, 15.6584069961, 16.3587362576, 15.6237273737, 16.1908200999, 17.3316213215, 16.5526329817 ], "latency_us": [ 289.667, 302.5, 258.833, 217.333, 254.666, 275.583, 244.542, 301.125, 235.208 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 60, "normalized_kernel_cost": 240, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 988500, "compute_span_ns": 988500, "calibration_cpu_ns": 1461875, "calibration_gpu_ticks": 1461875, "compute_passes": 1, "command_buffer_ns": 988583.371509, "command_buffers": 2 }, { "compute_ns": 1006792, "compute_span_ns": 1006792, "calibration_cpu_ns": 1695875, "calibration_gpu_ticks": 1695875, "compute_passes": 1, "command_buffer_ns": 1006833.33585, "command_buffers": 2 }, { "compute_ns": 1016917, "compute_span_ns": 1016917, "calibration_cpu_ns": 1718417, "calibration_gpu_ticks": 1718417, "compute_passes": 1, "command_buffer_ns": 1016958.38384, "command_buffers": 2 }, { "compute_ns": 957041, "compute_span_ns": 957041, "calibration_cpu_ns": 1473833, "calibration_gpu_ticks": 1473833, "compute_passes": 1, "command_buffer_ns": 957000.011113, "command_buffers": 2 }, { "compute_ns": 940292, "compute_span_ns": 940292, "calibration_cpu_ns": 1393541, "calibration_gpu_ticks": 1393541, "compute_passes": 1, "command_buffer_ns": 940374.971833, "command_buffers": 2 }, { "compute_ns": 953125, "compute_span_ns": 953125, "calibration_cpu_ns": 1464834, "calibration_gpu_ticks": 1464834, "compute_passes": 1, "command_buffer_ns": 953166.687395, "command_buffers": 2 }, { "compute_ns": 960625, "compute_span_ns": 960625, "calibration_cpu_ns": 1424708, "calibration_gpu_ticks": 1424708, "compute_passes": 1, "command_buffer_ns": 960708.304774, "command_buffers": 2 }, { "compute_ns": 1013375, "compute_span_ns": 1013375, "calibration_cpu_ns": 1833208, "calibration_gpu_ticks": 1833208, "compute_passes": 1, "command_buffer_ns": 1013499.97567, "command_buffers": 2 }, { "compute_ns": 986750, "compute_span_ns": 986750, "calibration_cpu_ns": 1749834, "calibration_gpu_ticks": 1749834, "compute_passes": 1, "command_buffer_ns": 986749.946605, "command_buffers": 2 } ], "latency": [ { "compute_ns": 17000, "compute_span_ns": 17000, "calibration_cpu_ns": 555125, "calibration_gpu_ticks": 555125, "compute_passes": 1, "command_buffer_ns": 17000.0130311, "command_buffers": 2 }, { "compute_ns": 19334, "compute_span_ns": 19334, "calibration_cpu_ns": 443042, "calibration_gpu_ticks": 443042, "compute_passes": 1, "command_buffer_ns": 19458.3553821, "command_buffers": 2 }, { "compute_ns": 19333, "compute_span_ns": 19333, "calibration_cpu_ns": 460500, "calibration_gpu_ticks": 460500, "compute_passes": 1, "command_buffer_ns": 19333.3253264, "command_buffers": 2 }, { "compute_ns": 19292, "compute_span_ns": 19292, "calibration_cpu_ns": 497625, "calibration_gpu_ticks": 497625, "compute_passes": 1, "command_buffer_ns": 19375.0020117, "command_buffers": 2 }, { "compute_ns": 24625, "compute_span_ns": 24625, "calibration_cpu_ns": 700458, "calibration_gpu_ticks": 700458, "compute_passes": 1, "command_buffer_ns": 24666.718673, "command_buffers": 2 }, { "compute_ns": 17583, "compute_span_ns": 17583, "calibration_cpu_ns": 622834, "calibration_gpu_ticks": 622834, "compute_passes": 1, "command_buffer_ns": 17583.3120011, "command_buffers": 2 }, { "compute_ns": 19458, "compute_span_ns": 19458, "calibration_cpu_ns": 445542, "calibration_gpu_ticks": 445542, "compute_passes": 1, "command_buffer_ns": 19499.9738596, "command_buffers": 2 }, { "compute_ns": 26375, "compute_span_ns": 26375, "calibration_cpu_ns": 913458, "calibration_gpu_ticks": 913458, "compute_passes": 1, "command_buffer_ns": 26374.9971054, "command_buffers": 2 }, { "compute_ns": 19792, "compute_span_ns": 19792, "calibration_cpu_ns": 409375, "calibration_gpu_ticks": 409375, "compute_passes": 1, "command_buffer_ns": 19791.6524485, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 991000.037175, "command_buffers": 2 }, { "command_buffer_ns": 957624.986768, "command_buffers": 2 }, { "command_buffer_ns": 957000.011113, "command_buffers": 2 }, { "command_buffer_ns": 961875.019129, "command_buffers": 2 }, { "command_buffer_ns": 1017583.30129, "command_buffers": 2 }, { "command_buffer_ns": 940125.028137, "command_buffers": 2 }, { "command_buffer_ns": 974916.678388, "command_buffers": 2 }, { "command_buffer_ns": 963458.383922, "command_buffers": 2 }, { "command_buffer_ns": 957291.689701, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 16791.629605, "command_buffers": 2 }, { "command_buffer_ns": 24666.718673, "command_buffers": 2 }, { "command_buffer_ns": 19583.3272301, "command_buffers": 2 }, { "command_buffer_ns": 19250.0301637, "command_buffers": 2 }, { "command_buffer_ns": 19500.0320673, "command_buffers": 2 }, { "command_buffer_ns": 18958.3515748, "command_buffers": 2 }, { "command_buffer_ns": 19250.0301637, "command_buffers": 2 }, { "command_buffer_ns": 19208.3534785, "command_buffers": 2 }, { "command_buffer_ns": 19208.3534785, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.484375580859375, 14.96289041825, 14.953125173640625, 15.029297173890626, 15.89973908265625, 14.689453564640626, 15.2330730998125, 15.054037248781249, 14.957682651578125 ], "command_buffer_throughput_us_p50": 15.029297173890626, "command_buffer_throughput_us_p90": 15.56744828121875, "command_buffer_latency_us": [ 16.791629604999997, 24.666718673, 19.5833272301, 19.2500301637, 19.500032067299998, 18.958351574799998, 19.2500301637, 19.2083534785, 19.2083534785 ], "command_buffer_latency_us_p50": 19.2500301637, "command_buffer_latency_us_p90": 20.60000551868 }, "compute_throughput_us": [ 15.4453125, 15.731125, 15.889328125, 14.953765625, 14.6920625, 14.892578125, 15.009765625, 15.833984375, 15.41796875 ], "compute_throughput_us_p50": 15.41796875, "compute_throughput_us_p90": 15.845053125, "command_buffer_throughput_us": [ 15.446615179828125, 15.73177087265625, 15.8899747475, 14.953125173640625, 14.693358934890624, 14.893229490546876, 15.01106726209375, 15.835937119843749, 15.417967915703125 ], "command_buffer_throughput_us_p50": 15.417967915703125, "command_buffer_throughput_us_p90": 15.846744645374999, "compute_latency_us": [ 17.0, 19.334, 19.333, 19.292, 24.625, 17.583, 19.458, 26.375, 19.792 ], "compute_latency_us_p50": 19.334, "compute_latency_us_p90": 24.975, "command_buffer_latency_us": [ 17.0000130311, 19.4583553821, 19.3333253264, 19.375002011699998, 24.666718673, 17.583312001099998, 19.499973859599997, 26.374997105400002, 19.791652448500002 ], "command_buffer_latency_us_p50": 19.4583553821, "command_buffer_latency_us_p90": 25.008374359479998, "counter_control_throughput_ratio": 1.0138722604628918, "counter_control_latency_ratio": 1.0124099584734736 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 16.1908200999, "throughput_us_p90": 16.70843064966, "latency_us_p50": 258.833, "latency_us_p90": 301.4 }, "torch": { "cold_call_ms": 5.662, "warmup_ms": 200.049125, "repetitions": 1432, "throughput_us": [ 20.356639664804472, 20.388617318435752, 20.570879888268156, 20.325418994413408, 20.291986731843576, 19.886900139664807, 20.035323324022347, 21.34843505586592, 20.309648743016762 ], "latency_us": [ 413.792, 247.958, 264.75, 264.875, 282.58299999999997, 225.5, 241.875, 237.625, 256.666 ], "throughput_us_p50": 20.325418994413408, "throughput_us_p90": 20.726390921787708, "latency_us_p50": 256.666, "latency_us_p90": 308.8248, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1152458.309661597, "command_buffers": 1 }, { "command_buffer_ns": 1134791.6442900896, "command_buffers": 1 }, { "command_buffer_ns": 1033874.9852962792, "command_buffers": 1 }, { "command_buffer_ns": 1033458.3348594606, "command_buffers": 1 }, { "command_buffer_ns": 1053416.6358411312, "command_buffers": 1 }, { "command_buffer_ns": 1093833.358027041, "command_buffers": 1 }, { "command_buffer_ns": 1030624.9605491757, "command_buffers": 1 }, { "command_buffer_ns": 1110750.0176876783, "command_buffers": 1 }, { "command_buffer_ns": 1147250.0045783818, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 18624.996300786734, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 24000.00812485814, "command_buffers": 1 }, { "command_buffer_ns": 26791.647542268038, "command_buffers": 1 }, { "command_buffer_ns": 21083.338651806116, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 20874.955225735903, "command_buffers": 1 }, { "command_buffer_ns": 21625.019144266844, "command_buffers": 1 }, { "command_buffer_ns": 21124.957129359245, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 18.007161088462453, 17.73111944203265, 16.154296645254362, 16.14778648217907, 16.459634935017675, 17.091146219172515, 16.10351500858087, 17.355469026369974, 17.925781321537215 ], "command_buffer_throughput_us_p50": 17.091146219172515, "command_buffer_throughput_us_p90": 17.942057274922263, "command_buffer_latency_us": [ 18.624996300786734, 20.99998528137803, 24.00000812485814, 26.791647542268038, 21.083338651806116, 20.99998528137803, 20.874955225735903, 21.625019144266844, 21.124957129359245 ], "command_buffer_latency_us_p50": 21.083338651806116, "command_buffer_latency_us_p90": 24.55833600834012 }, "throughput": [ { "compute_ns": 1115542.0, "compute_span_ns": 1115542.0, "command_buffer_ns": 1115583.349019289, "calibration_cpu_ns": 1815917.0, "calibration_gpu_ticks": 1815917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1111334.0, "compute_span_ns": 1111334.0, "command_buffer_ns": 1111374.9933429062, "calibration_cpu_ns": 2359834.0, "calibration_gpu_ticks": 2359834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1057125.0, "compute_span_ns": 1057125.0, "command_buffer_ns": 1057125.0459179282, "calibration_cpu_ns": 2053667.0, "calibration_gpu_ticks": 2053667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1129916.0, "compute_span_ns": 1129916.0, "command_buffer_ns": 1129875.0177957118, "calibration_cpu_ns": 1814875.0, "calibration_gpu_ticks": 1814875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1031625.0, "compute_span_ns": 1031625.0, "command_buffer_ns": 1031625.02637133, "calibration_cpu_ns": 1677500.0, "calibration_gpu_ticks": 1677500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1036917.0, "compute_span_ns": 1036917.0, "command_buffer_ns": 1036958.3033025265, "calibration_cpu_ns": 1716500.0, "calibration_gpu_ticks": 1716500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1038709.0, "compute_span_ns": 1038709.0, "command_buffer_ns": 1038749.9933131039, "calibration_cpu_ns": 1947708.0, "calibration_gpu_ticks": 1947708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1153042.0, "compute_span_ns": 1153042.0, "command_buffer_ns": 1153041.6668392718, "calibration_cpu_ns": 2111041.0, "calibration_gpu_ticks": 2111041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1147625.0, "compute_span_ns": 1147625.0, "command_buffer_ns": 1147624.9783299863, "calibration_cpu_ns": 1767875.0, "calibration_gpu_ticks": 1767875.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 28917.0, "compute_span_ns": 28917.0, "command_buffer_ns": 28916.634619235992, "calibration_cpu_ns": 935917.0, "calibration_gpu_ticks": 935917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21541.0, "compute_span_ns": 21541.0, "command_buffer_ns": 21499.989088624716, "calibration_cpu_ns": 721875.0, "calibration_gpu_ticks": 721875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21041.0, "compute_span_ns": 21041.0, "command_buffer_ns": 21000.043489038944, "calibration_cpu_ns": 520875.0, "calibration_gpu_ticks": 520875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20958.0, "compute_span_ns": 20958.0, "command_buffer_ns": 20958.3668038249, "calibration_cpu_ns": 435458.0, "calibration_gpu_ticks": 435458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20916.0, "compute_span_ns": 20916.0, "command_buffer_ns": 20874.955225735903, "calibration_cpu_ns": 497417.0, "calibration_gpu_ticks": 497417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21125.0, "compute_span_ns": 21125.0, "command_buffer_ns": 21124.957129359245, "calibration_cpu_ns": 441917.0, "calibration_gpu_ticks": 441917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 30416.0, "compute_span_ns": 30416.0, "command_buffer_ns": 30375.02756342292, "calibration_cpu_ns": 811792.0, "calibration_gpu_ticks": 811792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21416.0, "compute_span_ns": 21416.0, "command_buffer_ns": 21374.959032982588, "calibration_cpu_ns": 638542.0, "calibration_gpu_ticks": 638542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21167.0, "compute_span_ns": 21167.0, "command_buffer_ns": 21166.633814573288, "calibration_cpu_ns": 429167.0, "calibration_gpu_ticks": 429167.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 17.43034375, 17.36459375, 16.517578125, 17.6549375, 16.119140625, 16.201828125, 16.229828125, 18.01628125, 17.931640625 ], "compute_throughput_us_p50": 17.36459375, "compute_throughput_us_p90": 17.94856875, "command_buffer_throughput_us": [ 17.43098982842639, 17.36523427098291, 16.51757884246763, 17.654297153057996, 16.11914103705203, 16.202473489101976, 16.23046864551725, 18.01627604436362, 17.931640286406036 ], "command_buffer_throughput_us_p50": 17.36523427098291, "command_buffer_throughput_us_p90": 17.948567437997553, "compute_latency_us": [ 28.917, 21.541, 21.041, 20.958, 20.916, 21.125, 30.416, 21.416, 21.167 ], "compute_latency_us_p50": 21.167, "compute_latency_us_p90": 29.216800000000003, "command_buffer_latency_us": [ 28.916634619235992, 21.499989088624716, 21.000043489038944, 20.9583668038249, 20.874955225735903, 21.124957129359245, 30.37502756342292, 21.374959032982588, 21.166633814573288 ], "command_buffer_latency_us_p50": 21.166633814573288, "command_buffer_latency_us_p90": 29.208313208073378, "counter_control_throughput_ratio": 1.0003268457181156, "counter_control_latency_ratio": 1.001972864842226 }, "download_ms": 0.341709, "allocation_upload_ms": 1.4285, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7965798935977737, "device_compute_slowdown": 0.8878968878842903, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.288875, "warmup_ms": 200.420584, "repetitions": 736, "throughput_us": [ 23.136323369565222, 20.81148097826087, 22.582258152173914, 20.159703804347824, 20.377661684782606, 20.809158967391305, 20.808990489130434, 20.504642663043477, 21.930254076086957 ], "latency_us": [ 310.75, 373.584, 461.625, 335.583, 273.084, 260.29200000000003, 242.125, 255.91599999999997, 257.084 ], "throughput_us_p50": 20.809158967391305, "throughput_us_p90": 22.693071195652177, "latency_us_p50": 273.084, "latency_us_p90": 391.1922, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1095124.9860227108, "command_buffers": 1 }, { "command_buffer_ns": 1033583.3649151027, "command_buffers": 1 }, { "command_buffer_ns": 1071624.9817050993, "command_buffers": 1 }, { "command_buffer_ns": 1049583.3703316748, "command_buffers": 1 }, { "command_buffer_ns": 1028999.9772794545, "command_buffers": 1 }, { "command_buffer_ns": 1146750.000771135, "command_buffers": 1 }, { "command_buffer_ns": 1035749.9704696238, "command_buffers": 1 }, { "command_buffer_ns": 1059208.3563096821, "command_buffers": 1 }, { "command_buffer_ns": 1095500.0179819763, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 19125.00010803342, "command_buffers": 1 }, { "command_buffer_ns": 23874.978069216013, "command_buffers": 1 }, { "command_buffer_ns": 21375.0172406435, "command_buffers": 1 }, { "command_buffer_ns": 21000.043489038944, "command_buffers": 1 }, { "command_buffer_ns": 20916.69011861086, "command_buffers": 1 }, { "command_buffer_ns": 21333.282347768545, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 28625.01423805952, "command_buffers": 1 }, { "command_buffer_ns": 25541.69623181224, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 17.111327906604856, 16.14974007679848, 16.744140339142177, 16.39974016143242, 16.078124644991476, 17.917968762048986, 16.183593288587872, 16.550130567338783, 17.11718778096838 ], "command_buffer_throughput_us_p50": 16.550130567338783, "command_buffer_throughput_us_p90": 17.2773439771845, "command_buffer_latency_us": [ 19.12500010803342, 23.874978069216013, 21.3750172406435, 21.000043489038944, 20.91669011861086, 21.333282347768545, 20.99998528137803, 28.62501423805952, 25.54169623181224 ], "command_buffer_latency_us_p50": 21.333282347768545, "command_buffer_latency_us_p90": 26.158359833061695 }, "throughput": [ { "compute_ns": 1049417.0, "compute_span_ns": 1049417.0, "command_buffer_ns": 1049458.3402760327, "calibration_cpu_ns": 1986292.0, "calibration_gpu_ticks": 1986292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1105500.0, "compute_span_ns": 1105500.0, "command_buffer_ns": 1105499.9777115881, "calibration_cpu_ns": 1820500.0, "calibration_gpu_ticks": 1820500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1120125.0, "compute_span_ns": 1120125.0, "command_buffer_ns": 1120125.0017620623, "calibration_cpu_ns": 1876458.0, "calibration_gpu_ticks": 1876458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1054625.0, "compute_span_ns": 1054625.0, "command_buffer_ns": 1054625.0268816948, "calibration_cpu_ns": 1681459.0, "calibration_gpu_ticks": 1681459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1037334.0, "compute_span_ns": 1037334.0, "command_buffer_ns": 1037375.011947006, "calibration_cpu_ns": 1712667.0, "calibration_gpu_ticks": 1712667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1072500.0, "compute_span_ns": 1072500.0, "command_buffer_ns": 1072499.9592639506, "calibration_cpu_ns": 2074209.0, "calibration_gpu_ticks": 2074209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1042166.0, "compute_span_ns": 1042166.0, "command_buffer_ns": 1042124.9899081886, "calibration_cpu_ns": 1901125.0, "calibration_gpu_ticks": 1901125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1163500.0, "compute_span_ns": 1163500.0, "command_buffer_ns": 1163500.0118985772, "calibration_cpu_ns": 1837375.0, "calibration_gpu_ticks": 1837375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1156833.0, "compute_span_ns": 1156833.0, "command_buffer_ns": 1156791.695393622, "calibration_cpu_ns": 1789375.0, "calibration_gpu_ticks": 1789375.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 25209.0, "compute_span_ns": 25209.0, "command_buffer_ns": 25250.017642974854, "calibration_cpu_ns": 684875.0, "calibration_gpu_ticks": 684875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 19083.0, "compute_span_ns": 19083.0, "command_buffer_ns": 19041.646737605333, "calibration_cpu_ns": 448750.0, "calibration_gpu_ticks": 448750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21333.0, "compute_span_ns": 21333.0, "command_buffer_ns": 21333.34055542946, "calibration_cpu_ns": 448458.0, "calibration_gpu_ticks": 448458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21125.0, "compute_span_ns": 21125.0, "command_buffer_ns": 21125.01533702016, "calibration_cpu_ns": 387750.0, "calibration_gpu_ticks": 387750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 30083.0, "compute_span_ns": 30083.0, "command_buffer_ns": 30083.29076692462, "calibration_cpu_ns": 1019583.0, "calibration_gpu_ticks": 1019583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21125.0, "compute_span_ns": 21125.0, "command_buffer_ns": 21125.01533702016, "calibration_cpu_ns": 623792.0, "calibration_gpu_ticks": 623792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21166.0, "compute_span_ns": 21166.0, "command_buffer_ns": 21125.01533702016, "calibration_cpu_ns": 559125.0, "calibration_gpu_ticks": 559125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20917.0, "compute_span_ns": 20917.0, "command_buffer_ns": 20916.69011861086, "calibration_cpu_ns": 479542.0, "calibration_gpu_ticks": 479542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21167.0, "compute_span_ns": 21167.0, "command_buffer_ns": 21166.6920222342, "calibration_cpu_ns": 447083.0, "calibration_gpu_ticks": 447083.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 16.397140625, 17.2734375, 17.501953125, 16.478515625, 16.20834375, 16.7578125, 16.28384375, 18.1796875, 18.075515625 ], "compute_throughput_us_p50": 16.7578125, "compute_throughput_us_p90": 18.09635, "command_buffer_throughput_us": [ 16.39778656681301, 17.273437151743565, 17.501953152532224, 16.47851604502648, 16.20898456167197, 16.757811863499228, 16.283202967315447, 18.17968768591527, 18.074870240525343 ], "command_buffer_throughput_us_p50": 16.757811863499228, "command_buffer_throughput_us_p90": 18.095833729603328, "compute_latency_us": [ 25.209, 19.083, 21.333, 21.125, 30.083, 21.125, 21.166, 20.917, 21.167 ], "compute_latency_us_p50": 21.166, "compute_latency_us_p90": 26.1838, "command_buffer_latency_us": [ 25.250017642974854, 19.041646737605333, 21.33334055542946, 21.12501533702016, 30.08329076692462, 21.12501533702016, 21.12501533702016, 20.91669011861086, 21.1666920222342 ], "command_buffer_latency_us_p50": 21.12501533702016, "command_buffer_latency_us_p90": 26.216672267764807, "counter_control_throughput_ratio": 1.008139003743901, "counter_control_latency_ratio": 0.9980502151298949 }, "download_ms": 0.370959, "allocation_upload_ms": 0.711208, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-74duy0bz/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b5a32df1046f38df4f8e9cbba5277df92f4beb4d4f9631ac4564ef5cda134264", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.079166, "capture_ms": 0.062542, "compile_ms": 111.46075, "allocation_upload_ms": 2.113042, "cold_call_ms": 3.600333, "warmup_ms": 200.200166, "download_ms": 0.8845, "repetitions": 1898, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 15.8496227608, 15.7541053741, 15.7845321391, 15.7956622761, 16.113869863, 16.0145985248, 15.7049083246, 15.9046364594, 15.7390458377 ], "latency_us": [ 1404.292, 631.375, 319.667, 258.041, 244.792, 238.834, 256.333, 228.834, 225.584 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 68, "normalized_kernel_cost": 272, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 937708, "compute_span_ns": 937708, "calibration_cpu_ns": 1874292, "calibration_gpu_ticks": 1874292, "compute_passes": 1, "command_buffer_ns": 937750.039157, "command_buffers": 2 }, { "compute_ns": 926583, "compute_span_ns": 926583, "calibration_cpu_ns": 1474875, "calibration_gpu_ticks": 1474875, "compute_passes": 1, "command_buffer_ns": 926625.041757, "command_buffers": 2 }, { "compute_ns": 952791, "compute_span_ns": 952791, "calibration_cpu_ns": 2122250, "calibration_gpu_ticks": 2122250, "compute_passes": 1, "command_buffer_ns": 952791.655436, "command_buffers": 2 }, { "compute_ns": 934750, "compute_span_ns": 934750, "calibration_cpu_ns": 1461334, "calibration_gpu_ticks": 1461334, "compute_passes": 1, "command_buffer_ns": 934874.988161, "command_buffers": 2 }, { "compute_ns": 1234042, "compute_span_ns": 1234042, "calibration_cpu_ns": 2200958, "calibration_gpu_ticks": 2200958, "compute_passes": 1, "command_buffer_ns": 1234166.67338, "command_buffers": 2 }, { "compute_ns": 940166, "compute_span_ns": 940166, "calibration_cpu_ns": 1429875, "calibration_gpu_ticks": 1429875, "compute_passes": 1, "command_buffer_ns": 940208.3233, "command_buffers": 2 }, { "compute_ns": 983708, "compute_span_ns": 983708, "calibration_cpu_ns": 1924875, "calibration_gpu_ticks": 1924875, "compute_passes": 1, "command_buffer_ns": 983750.040177, "command_buffers": 2 }, { "compute_ns": 917625, "compute_span_ns": 917625, "calibration_cpu_ns": 1397375, "calibration_gpu_ticks": 1397375, "compute_passes": 1, "command_buffer_ns": 917708.384804, "command_buffers": 2 }, { "compute_ns": 933125, "compute_span_ns": 933125, "calibration_cpu_ns": 1434500, "calibration_gpu_ticks": 1434500, "compute_passes": 1, "command_buffer_ns": 933208.269998, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18417, "compute_span_ns": 18417, "calibration_cpu_ns": 520500, "calibration_gpu_ticks": 520500, "compute_passes": 1, "command_buffer_ns": 18500.0244528, "command_buffers": 2 }, { "compute_ns": 18417, "compute_span_ns": 18417, "calibration_cpu_ns": 451542, "calibration_gpu_ticks": 451542, "compute_passes": 1, "command_buffer_ns": 18416.6710824, "command_buffers": 2 }, { "compute_ns": 25208, "compute_span_ns": 25208, "calibration_cpu_ns": 1125750, "calibration_gpu_ticks": 1125750, "compute_passes": 1, "command_buffer_ns": 25166.6642725, "command_buffers": 2 }, { "compute_ns": 18417, "compute_span_ns": 18417, "calibration_cpu_ns": 456125, "calibration_gpu_ticks": 456125, "compute_passes": 1, "command_buffer_ns": 18416.6710824, "command_buffers": 2 }, { "compute_ns": 18542, "compute_span_ns": 18542, "calibration_cpu_ns": 527500, "calibration_gpu_ticks": 527500, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "compute_ns": 25083, "compute_span_ns": 25083, "calibration_cpu_ns": 1381417, "calibration_gpu_ticks": 1381417, "compute_passes": 1, "command_buffer_ns": 25125.045795, "command_buffers": 2 }, { "compute_ns": 18458, "compute_span_ns": 18458, "calibration_cpu_ns": 462167, "calibration_gpu_ticks": 462167, "compute_passes": 1, "command_buffer_ns": 18458.3477676, "command_buffers": 2 }, { "compute_ns": 20458, "compute_span_ns": 20458, "calibration_cpu_ns": 540375, "calibration_gpu_ticks": 540375, "compute_passes": 1, "command_buffer_ns": 20500.0396818, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 409625, "calibration_gpu_ticks": 409625, "compute_passes": 1, "command_buffer_ns": 18541.701138, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 987166.713458, "command_buffers": 2 }, { "command_buffer_ns": 990125.001408, "command_buffers": 2 }, { "command_buffer_ns": 934666.662943, "command_buffers": 2 }, { "command_buffer_ns": 923749.990761, "command_buffers": 2 }, { "command_buffer_ns": 937083.386816, "command_buffers": 2 }, { "command_buffer_ns": 1208791.68389, "command_buffers": 2 }, { "command_buffer_ns": 912333.256565, "command_buffers": 2 }, { "command_buffer_ns": 958624.994382, "command_buffers": 2 }, { "command_buffer_ns": 908958.376385, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 23833.3595917, "command_buffers": 2 }, { "command_buffer_ns": 18374.9943972, "command_buffers": 2 }, { "command_buffer_ns": 18291.6410267, "command_buffers": 2 }, { "command_buffer_ns": 18416.6710824, "command_buffers": 2 }, { "command_buffer_ns": 18583.3196156, "command_buffers": 2 }, { "command_buffer_ns": 18249.9643415, "command_buffers": 2 }, { "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "command_buffer_ns": 26750.0290647, "command_buffers": 2 }, { "command_buffer_ns": 18583.3778232, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.42447989778125, 15.470703147, 14.604166608484375, 14.433593605640624, 14.641927919, 18.88737006078125, 14.255207133828126, 14.97851553721875, 14.202474631015624 ], "command_buffer_throughput_us_p50": 14.641927919, "command_buffer_throughput_us_p90": 16.15403652975625, "command_buffer_latency_us": [ 23.8333595917, 18.374994397200002, 18.2916410267, 18.4166710824, 18.5833196156, 18.2499643415, 18.624996300800003, 26.7500290647, 18.5833778232 ], "command_buffer_latency_us_p50": 18.5833196156, "command_buffer_latency_us_p90": 24.4166934863 }, "compute_throughput_us": [ 14.6516875, 14.477859375, 14.887359375, 14.60546875, 19.28190625, 14.69009375, 15.3704375, 14.337890625, 14.580078125 ], "compute_throughput_us_p50": 14.6516875, "compute_throughput_us_p90": 16.15273125, "command_buffer_throughput_us": [ 14.652344361828126, 14.478516277453126, 14.887369616187499, 14.607421690015624, 19.2838542715625, 14.690755051562501, 15.371094377765624, 14.3391935125625, 14.58137921871875 ], "command_buffer_throughput_us_p50": 14.652344361828126, "command_buffer_throughput_us_p90": 16.153646356525, "compute_latency_us": [ 18.417, 18.417, 25.208, 18.417, 18.542, 25.083, 18.458, 20.458, 18.5 ], "compute_latency_us_p50": 18.5, "compute_latency_us_p90": 25.107999999999997, "command_buffer_latency_us": [ 18.5000244528, 18.4166710824, 25.166664272499997, 18.4166710824, 18.541642930400002, 25.125045795000002, 18.4583477676, 20.5000396818, 18.541701138 ], "command_buffer_latency_us_p50": 18.541642930400002, "command_buffer_latency_us_p90": 25.1333694905, "counter_control_throughput_ratio": 1.0120432990649721, "counter_control_latency_ratio": 0.9977573137888867 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 15.7956622761, "throughput_us_p90": 16.03445279244, "latency_us_p50": 256.333, "latency_us_p90": 785.9584000000001 }, "slowdown": 0.759072593988655, "device_compute_slowdown": 0.8743198135198135, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-enih3m58/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "232522da57db77c64b377ec2facdc2e1df33cd57064a4297b89d6582fd4ad63b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.710333, "capture_ms": 0.065, "compile_ms": 54.660542, "allocation_upload_ms": 6.511833, "cold_call_ms": 5.2535, "warmup_ms": 200.612917, "download_ms": 3.524833, "repetitions": 332, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 64.2206325301, 65.8682228916, 65.9956084337, 64.7289156627, 65.6670421687, 63.7980692771, 65.7510060241, 66.4986204819, 65.5813253012 ], "latency_us": [ 376.583, 338.375, 309.042, 300.875, 280.667, 1484.125, 329.458, 300.584, 266.833 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 104, "normalized_kernel_cost": 1248, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3838084, "compute_span_ns": 3838084, "calibration_cpu_ns": 4576042, "calibration_gpu_ticks": 4576042, "compute_passes": 1, "command_buffer_ns": 3838166.65271, "command_buffers": 2 }, { "compute_ns": 3904709, "compute_span_ns": 3904709, "calibration_cpu_ns": 4587666, "calibration_gpu_ticks": 4587666, "compute_passes": 1, "command_buffer_ns": 3904833.28374, "command_buffers": 2 }, { "compute_ns": 3932125, "compute_span_ns": 3932125, "calibration_cpu_ns": 4830250, "calibration_gpu_ticks": 4830250, "compute_passes": 1, "command_buffer_ns": 3932249.96515, "command_buffers": 2 }, { "compute_ns": 3886541, "compute_span_ns": 3886541, "calibration_cpu_ns": 4707167, "calibration_gpu_ticks": 4707167, "compute_passes": 1, "command_buffer_ns": 3886541.70092, "command_buffers": 2 }, { "compute_ns": 3896500, "compute_span_ns": 3896500, "calibration_cpu_ns": 4751208, "calibration_gpu_ticks": 4751208, "compute_passes": 1, "command_buffer_ns": 3896541.66065, "command_buffers": 2 }, { "compute_ns": 3948500, "compute_span_ns": 3948500, "calibration_cpu_ns": 4774208, "calibration_gpu_ticks": 4774208, "compute_passes": 1, "command_buffer_ns": 3948541.70736, "command_buffers": 2 }, { "compute_ns": 3911709, "compute_span_ns": 3911709, "calibration_cpu_ns": 4588625, "calibration_gpu_ticks": 4588625, "compute_passes": 1, "command_buffer_ns": 3911749.98367, "command_buffers": 2 }, { "compute_ns": 3845541, "compute_span_ns": 3845541, "calibration_cpu_ns": 4579500, "calibration_gpu_ticks": 4579500, "compute_passes": 1, "command_buffer_ns": 3845541.67977, "command_buffers": 2 }, { "compute_ns": 3864125, "compute_span_ns": 3864125, "calibration_cpu_ns": 4589375, "calibration_gpu_ticks": 4589375, "compute_passes": 1, "command_buffer_ns": 3864124.99938, "command_buffers": 2 } ], "latency": [ { "compute_ns": 88416, "compute_span_ns": 88416, "calibration_cpu_ns": 697458, "calibration_gpu_ticks": 697458, "compute_passes": 1, "command_buffer_ns": 88416.680228, "command_buffers": 2 }, { "compute_ns": 120125, "compute_span_ns": 120125, "calibration_cpu_ns": 718000, "calibration_gpu_ticks": 718000, "compute_passes": 1, "command_buffer_ns": 120166.689157, "command_buffers": 2 }, { "compute_ns": 114709, "compute_span_ns": 114709, "calibration_cpu_ns": 986709, "calibration_gpu_ticks": 986709, "compute_passes": 1, "command_buffer_ns": 114750.000648, "command_buffers": 2 }, { "compute_ns": 69000, "compute_span_ns": 69000, "calibration_cpu_ns": 461208, "calibration_gpu_ticks": 461208, "compute_passes": 1, "command_buffer_ns": 69083.3549015, "command_buffers": 2 }, { "compute_ns": 75917, "compute_span_ns": 75917, "calibration_cpu_ns": 681792, "calibration_gpu_ticks": 681792, "compute_passes": 1, "command_buffer_ns": 76041.6733101, "command_buffers": 2 }, { "compute_ns": 113584, "compute_span_ns": 113584, "calibration_cpu_ns": 1041458, "calibration_gpu_ticks": 1041458, "compute_passes": 1, "command_buffer_ns": 113708.258141, "command_buffers": 2 }, { "compute_ns": 80625, "compute_span_ns": 80625, "calibration_cpu_ns": 486333, "calibration_gpu_ticks": 486333, "compute_passes": 1, "command_buffer_ns": 80666.6794233, "command_buffers": 2 }, { "compute_ns": 71583, "compute_span_ns": 71583, "calibration_cpu_ns": 495208, "calibration_gpu_ticks": 495208, "compute_passes": 1, "command_buffer_ns": 71541.6972525, "command_buffers": 2 }, { "compute_ns": 92917, "compute_span_ns": 92917, "calibration_cpu_ns": 625750, "calibration_gpu_ticks": 625750, "compute_passes": 1, "command_buffer_ns": 93000.009656, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3936249.9956, "command_buffers": 2 }, { "command_buffer_ns": 3842958.30736, "command_buffers": 2 }, { "command_buffer_ns": 3915916.66266, "command_buffers": 2 }, { "command_buffer_ns": 3875916.59091, "command_buffers": 2 }, { "command_buffer_ns": 3834875.00949, "command_buffers": 2 }, { "command_buffer_ns": 3866999.99217, "command_buffers": 2 }, { "command_buffer_ns": 3875416.70352, "command_buffers": 2 }, { "command_buffer_ns": 3838833.30505, "command_buffers": 2 }, { "command_buffer_ns": 3878708.28854, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 112624.955364, "command_buffers": 2 }, { "command_buffer_ns": 92958.3329707, "command_buffers": 2 }, { "command_buffer_ns": 69166.6500643, "command_buffers": 2 }, { "command_buffer_ns": 67499.9901094, "command_buffers": 2 }, { "command_buffer_ns": 64874.9992251, "command_buffers": 2 }, { "command_buffer_ns": 112875.015475, "command_buffers": 2 }, { "command_buffer_ns": 78291.6904427, "command_buffers": 2 }, { "command_buffer_ns": 109208.340291, "command_buffers": 2 }, { "command_buffer_ns": 98083.3428912, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 61.50390618125, 60.0462235525, 61.1861978540625, 60.561196732968746, 59.91992202328125, 60.42187487765625, 60.5533859925, 59.98177039140625, 60.6048170084375 ], "command_buffer_throughput_us_p50": 60.5533859925, "command_buffer_throughput_us_p90": 61.2497395195, "command_buffer_latency_us": [ 112.62495536399999, 92.9583329707, 69.1666500643, 67.49999010939999, 64.8749992251, 112.875015475, 78.2916904427, 109.208340291, 98.08334289119999 ], "command_buffer_latency_us_p50": 92.9583329707, "command_buffer_latency_us_p90": 112.6749673862 }, "compute_throughput_us": [ 59.9700625, 61.011078125, 61.439453125, 60.727203125, 60.8828125, 61.6953125, 61.120453125, 60.086578125, 60.376953125 ], "compute_throughput_us_p50": 60.8828125, "compute_throughput_us_p90": 61.490625, "command_buffer_throughput_us": [ 59.97135394859375, 61.0130200584375, 61.44140570546875, 60.727214076875, 60.88346344765625, 61.695964177499995, 61.12109349484375, 60.08658874640625, 60.376953115312496 ], "command_buffer_throughput_us_p50": 60.88346344765625, "command_buffer_throughput_us_p90": 61.492317399875, "compute_latency_us": [ 88.416, 120.125, 114.709, 69.0, 75.917, 113.584, 80.625, 71.583, 92.917 ], "compute_latency_us_p50": 88.416, "compute_latency_us_p90": 115.79220000000001, "command_buffer_latency_us": [ 88.416680228, 120.16668915700001, 114.750000648, 69.0833549015, 76.0416733101, 113.708258141, 80.6666794233, 71.5416972525, 93.00000965599999 ], "command_buffer_latency_us_p50": 88.416680228, "command_buffer_latency_us_p90": 115.8333383498, "counter_control_throughput_ratio": 1.0041710035981475, "counter_control_latency_ratio": 1.023457259616391 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 65.6670421687, "throughput_us_p90": 66.09621084334, "latency_us_p50": 309.042, "latency_us_p90": 598.0914000000002 }, "torch": { "cold_call_ms": 9.931084, "warmup_ms": 200.189583, "repetitions": 255, "throughput_us": [ 84.40588235294118, 84.81421568627451, 86.78104705882352, 86.2581725490196, 85.10865882352942, 83.29166666666667, 86.48055294117647, 85.73006666666666, 84.2063725490196 ], "latency_us": [ 496.917, 335.334, 323.709, 911.25, 410.04200000000003, 337.084, 402.75, 612.917, 474.917 ], "throughput_us_p50": 85.10865882352942, "throughput_us_p90": 86.54065176470588, "latency_us_p50": 410.04200000000003, "latency_us_p90": 672.5836, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5012416.688259691, "command_buffers": 1 }, { "command_buffer_ns": 4981875.012163073, "command_buffers": 1 }, { "command_buffer_ns": 4970499.954652041, "command_buffers": 1 }, { "command_buffer_ns": 5020458.367653191, "command_buffers": 1 }, { "command_buffer_ns": 4970041.6857376695, "command_buffers": 1 }, { "command_buffer_ns": 5102625.000290573, "command_buffers": 1 }, { "command_buffer_ns": 5033958.35403353, "command_buffers": 1 }, { "command_buffer_ns": 5030791.6244491935, "command_buffers": 1 }, { "command_buffer_ns": 5075083.347037435, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 114625.0288002193, "command_buffers": 1 }, { "command_buffer_ns": 84583.3565108478, "command_buffers": 1 }, { "command_buffer_ns": 86374.98831376433, "command_buffers": 1 }, { "command_buffer_ns": 106874.96978789568, "command_buffers": 1 }, { "command_buffer_ns": 85124.97879564762, "command_buffers": 1 }, { "command_buffer_ns": 117374.99153241515, "command_buffers": 1 }, { "command_buffer_ns": 85874.98450651765, "command_buffers": 1 }, { "command_buffer_ns": 79624.995123595, "command_buffers": 1 }, { "command_buffer_ns": 84125.02938881516, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 78.31901075405767, 77.84179706504801, 77.66406179143814, 78.44466199458111, 77.65690133965109, 79.7285156295402, 78.6555992817739, 78.60611913201865, 79.29817729745992 ], "command_buffer_throughput_us_p50": 78.44466199458111, "command_buffer_throughput_us_p90": 79.38424496387597, "command_buffer_latency_us": [ 114.6250288002193, 84.5833565108478, 86.37498831376433, 106.87496978789568, 85.12497879564762, 117.37499153241515, 85.87498450651765, 79.624995123595, 84.12502938881516 ], "command_buffer_latency_us_p50": 85.87498450651765, "command_buffer_latency_us_p90": 115.17502134665847 }, "throughput": [ { "compute_ns": 4922000.0, "compute_span_ns": 4922000.0, "command_buffer_ns": 4921999.992802739, "calibration_cpu_ns": 6332166.0, "calibration_gpu_ticks": 6332166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5048750.0, "compute_span_ns": 5048750.0, "command_buffer_ns": 5048749.968409538, "calibration_cpu_ns": 6150250.0, "calibration_gpu_ticks": 6150250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5078083.0, "compute_span_ns": 5078083.0, "command_buffer_ns": 5078041.693195701, "calibration_cpu_ns": 6240417.0, "calibration_gpu_ticks": 6240417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5012791.0, "compute_span_ns": 5012791.0, "command_buffer_ns": 5012749.9853260815, "calibration_cpu_ns": 6031125.0, "calibration_gpu_ticks": 6031125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5108208.0, "compute_span_ns": 5108208.0, "command_buffer_ns": 5108166.7188555, "calibration_cpu_ns": 6021834.0, "calibration_gpu_ticks": 6021834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5155958.0, "compute_span_ns": 5155958.0, "command_buffer_ns": 5155958.351679146, "calibration_cpu_ns": 6234875.0, "calibration_gpu_ticks": 6234875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5072584.0, "compute_span_ns": 5072584.0, "command_buffer_ns": 5072625.004686415, "calibration_cpu_ns": 6010667.0, "calibration_gpu_ticks": 6010667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5002000.0, "compute_span_ns": 5002000.0, "command_buffer_ns": 5002000.0198856, "calibration_cpu_ns": 6047334.0, "calibration_gpu_ticks": 6047334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5103666.0, "compute_span_ns": 5103666.0, "command_buffer_ns": 5103625.007905066, "calibration_cpu_ns": 6037250.0, "calibration_gpu_ticks": 6037250.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 113417.0, "compute_span_ns": 113417.0, "command_buffer_ns": 113458.31444486976, "calibration_cpu_ns": 880459.0, "calibration_gpu_ticks": 880459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 93584.0, "compute_span_ns": 93584.0, "command_buffer_ns": 93624.98531118035, "calibration_cpu_ns": 522958.0, "calibration_gpu_ticks": 522958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 114250.0, "compute_span_ns": 114250.0, "command_buffer_ns": 114249.99684095383, "calibration_cpu_ns": 788417.0, "calibration_gpu_ticks": 788417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 87333.0, "compute_span_ns": 87333.0, "command_buffer_ns": 87333.37745070457, "calibration_cpu_ns": 616833.0, "calibration_gpu_ticks": 616833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82083.0, "compute_span_ns": 82083.0, "command_buffer_ns": 82083.33747461438, "calibration_cpu_ns": 488417.0, "calibration_gpu_ticks": 488417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 79834.0, "compute_span_ns": 79834.0, "command_buffer_ns": 79874.99702721834, "calibration_cpu_ns": 615708.0, "calibration_gpu_ticks": 615708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 111291.0, "compute_span_ns": 111291.0, "command_buffer_ns": 111249.97399747372, "calibration_cpu_ns": 1285125.0, "calibration_gpu_ticks": 1285125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 99750.0, "compute_span_ns": 99750.0, "command_buffer_ns": 99750.00284612179, "calibration_cpu_ns": 569000.0, "calibration_gpu_ticks": 569000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 109542.0, "compute_span_ns": 109542.0, "command_buffer_ns": 109583.37225019932, "calibration_cpu_ns": 738459.0, "calibration_gpu_ticks": 738459.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 76.90625, 78.88671875, 79.345046875, 78.324859375, 79.81575, 80.56184375, 79.259125, 78.15625, 79.74478125 ], "compute_throughput_us_p50": 79.259125, "compute_throughput_us_p90": 79.96496875, "command_buffer_throughput_us": [ 76.9062498875428, 78.88671825639904, 79.34440145618282, 78.32421852072002, 79.81510498211719, 80.56184924498666, 79.25976569822524, 78.1562503107125, 79.74414074851666 ], "command_buffer_throughput_us_p50": 79.25976569822524, "command_buffer_throughput_us_p90": 79.96445383469108, "compute_latency_us": [ 113.417, 93.584, 114.25, 87.333, 82.083, 79.834, 111.291, 99.75, 109.542 ], "compute_latency_us_p50": 99.75, "compute_latency_us_p90": 113.5836, "command_buffer_latency_us": [ 113.45831444486976, 93.62498531118035, 114.24999684095383, 87.33337745070457, 82.08333747461438, 79.87499702721834, 111.24997399747372, 99.75000284612179, 109.58337225019932 ], "command_buffer_latency_us_p50": 99.75000284612179, "command_buffer_latency_us_p90": 113.61665092408657, "counter_control_throughput_ratio": 1.0076811622054647, "counter_control_latency_ratio": 1.1068960747516914 }, "download_ms": 0.788666, "allocation_upload_ms": 18.001209, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7715671128699007, "device_compute_slowdown": 0.7681489355326595, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.424541, "warmup_ms": 201.688375, "repetitions": 346, "throughput_us": [ 83.2047225433526, 83.58140751445087, 83.32526589595376, 82.82128901734103, 86.45351445086705, 82.66642485549133, 83.6447514450867, 82.55419075144509, 83.84344797687861 ], "latency_us": [ 437.20799999999997, 403.375, 324.33299999999997, 363.458, 321.79200000000003, 303.542, 307.125, 318.083, 300.25 ], "throughput_us_p50": 83.32526589595376, "throughput_us_p90": 84.3654612716763, "latency_us_p50": 321.79200000000003, "latency_us_p90": 410.1416, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4982500.046025962, "command_buffers": 1 }, { "command_buffer_ns": 4944708.3147242665, "command_buffers": 1 }, { "command_buffer_ns": 5029041.669331491, "command_buffers": 1 }, { "command_buffer_ns": 5024624.988436699, "command_buffers": 1 }, { "command_buffer_ns": 5016541.632357985, "command_buffers": 1 }, { "command_buffer_ns": 5086374.992970377, "command_buffers": 1 }, { "command_buffer_ns": 5047833.372373134, "command_buffers": 1 }, { "command_buffer_ns": 4986166.663002223, "command_buffers": 1 }, { "command_buffer_ns": 5050041.654612869, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 104458.30412209034, "command_buffers": 1 }, { "command_buffer_ns": 80750.0327937305, "command_buffers": 1 }, { "command_buffer_ns": 83749.9974295497, "command_buffers": 1 }, { "command_buffer_ns": 123375.03721937537, "command_buffers": 1 }, { "command_buffer_ns": 80875.00464171171, "command_buffers": 1 }, { "command_buffer_ns": 82083.33747461438, "command_buffers": 1 }, { "command_buffer_ns": 81125.00654533505, "command_buffers": 1 }, { "command_buffer_ns": 85124.97879564762, "command_buffers": 1 }, { "command_buffer_ns": 80375.00083446503, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 77.85156321915565, 77.26106741756666, 78.57877608330455, 78.50976544432342, 78.38346300559351, 79.47460926516214, 78.87239644333022, 77.90885410940973, 78.90690085332608 ], "command_buffer_throughput_us_p50": 78.50976544432342, "command_buffer_throughput_us_p90": 79.02044253569329, "command_buffer_latency_us": [ 104.45830412209034, 80.7500327937305, 83.7499974295497, 123.37503721937537, 80.87500464171171, 82.08333747461438, 81.12500654533505, 85.12497879564762, 80.37500083446503 ], "command_buffer_latency_us_p50": 82.08333747461438, "command_buffer_latency_us_p90": 108.24165074154735 }, "throughput": [ { "compute_ns": 5009625.0, "compute_span_ns": 5009625.0, "command_buffer_ns": 5009624.99063462, "calibration_cpu_ns": 6121500.0, "calibration_gpu_ticks": 6121500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4986916.0, "compute_span_ns": 4986916.0, "command_buffer_ns": 4986875.05023554, "calibration_cpu_ns": 5863375.0, "calibration_gpu_ticks": 5863375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4988459.0, "compute_span_ns": 4988459.0, "command_buffer_ns": 4988499.9752976, "calibration_cpu_ns": 5983750.0, "calibration_gpu_ticks": 5983750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5023583.0, "compute_span_ns": 5023583.0, "command_buffer_ns": 5023583.362344652, "calibration_cpu_ns": 5989791.0, "calibration_gpu_ticks": 5989791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5063500.0, "compute_span_ns": 5063500.0, "command_buffer_ns": 5063499.964307994, "calibration_cpu_ns": 6213416.0, "calibration_gpu_ticks": 6213416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5061667.0, "compute_span_ns": 5061667.0, "command_buffer_ns": 5061666.655819863, "calibration_cpu_ns": 6089584.0, "calibration_gpu_ticks": 6089584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4989666.0, "compute_span_ns": 4989666.0, "command_buffer_ns": 4989624.954760075, "calibration_cpu_ns": 5970500.0, "calibration_gpu_ticks": 5970500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4950791.0, "compute_span_ns": 4950791.0, "command_buffer_ns": 4950749.97888878, "calibration_cpu_ns": 5830750.0, "calibration_gpu_ticks": 5830750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4955291.0, "compute_span_ns": 4955291.0, "command_buffer_ns": 4955250.013154, "calibration_cpu_ns": 5949250.0, "calibration_gpu_ticks": 5949250.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 96750.0, "compute_span_ns": 96750.0, "command_buffer_ns": 96749.98000264168, "calibration_cpu_ns": 584000.0, "calibration_gpu_ticks": 584000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84709.0, "compute_span_ns": 84709.0, "command_buffer_ns": 84750.00504404306, "calibration_cpu_ns": 621542.0, "calibration_gpu_ticks": 621542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82500.0, "compute_span_ns": 82500.0, "command_buffer_ns": 82499.98791143298, "calibration_cpu_ns": 569375.0, "calibration_gpu_ticks": 569375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80417.0, "compute_span_ns": 80417.0, "command_buffer_ns": 80416.61931201816, "calibration_cpu_ns": 519042.0, "calibration_gpu_ticks": 519042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 90834.0, "compute_span_ns": 90834.0, "command_buffer_ns": 90875.0225789845, "calibration_cpu_ns": 926708.0, "calibration_gpu_ticks": 926708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80875.0, "compute_span_ns": 80875.0, "command_buffer_ns": 80874.9464340508, "calibration_cpu_ns": 486750.0, "calibration_gpu_ticks": 486750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 114291.0, "compute_span_ns": 114291.0, "command_buffer_ns": 114249.99684095383, "calibration_cpu_ns": 769458.0, "calibration_gpu_ticks": 769458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 86125.0, "compute_span_ns": 86125.0, "command_buffer_ns": 86124.98641014099, "calibration_cpu_ns": 847667.0, "calibration_gpu_ticks": 847667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82209.0, "compute_span_ns": 82209.0, "command_buffer_ns": 82249.98600780964, "calibration_cpu_ns": 525417.0, "calibration_gpu_ticks": 525417.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 78.275390625, 77.9205625, 77.944671875, 78.493484375, 79.1171875, 79.088546875, 77.96353125, 77.356109375, 77.426421875 ], "compute_throughput_us_p50": 77.96353125, "compute_throughput_us_p90": 79.09427500000001, "command_buffer_throughput_us": [ 78.27539047866594, 77.91992265993031, 77.945312114025, 78.4934900366352, 79.1171869423124, 79.08854149718536, 77.96288991812617, 77.35546842013719, 77.42578145553125 ], "command_buffer_throughput_us_p50": 77.96288991812617, "command_buffer_throughput_us_p90": 79.09427058621077, "compute_latency_us": [ 96.75, 84.709, 82.5, 80.417, 90.834, 80.875, 114.291, 86.125, 82.209 ], "compute_latency_us_p50": 84.709, "compute_latency_us_p90": 100.2582, "command_buffer_latency_us": [ 96.74998000264168, 84.75000504404306, 82.49998791143298, 80.41661931201816, 90.8750225789845, 80.8749464340508, 114.24999684095383, 86.12498641014099, 82.24998600780964 ], "command_buffer_latency_us_p50": 84.75000504404306, "command_buffer_latency_us_p90": 100.24998337030411, "counter_control_throughput_ratio": 0.9951422501910178, "counter_control_latency_ratio": 1.0117475226266313 }, "download_ms": 0.576416, "allocation_upload_ms": 1.694042, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-q3nq4zzf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "c9fd11968a5edbf77e894438f42e4bd465ad146629716a6f08eacac9ab82063f", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 23.226458, "capture_ms": 0.059875, "compile_ms": 90.198584, "allocation_upload_ms": 5.911041, "cold_call_ms": 10.272375, "warmup_ms": 200.627292, "download_ms": 2.558125, "repetitions": 425, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 66.88392, 66.7468635294, 66.6147058824, 68.0657835294, 67.1752941176, 67.1581364706, 66.78804, 66.7577435294, 68.38 ], "latency_us": [ 391.667, 312.75, 279.625, 292.042, 290.334, 740.334, 510.208, 1212.333, 332.75 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 96, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4199667, "compute_span_ns": 4199667, "calibration_cpu_ns": 4922584, "calibration_gpu_ticks": 4922584, "compute_passes": 1, "command_buffer_ns": 4199833.31813, "command_buffers": 2 }, { "compute_ns": 4221666, "compute_span_ns": 4221666, "calibration_cpu_ns": 4939625, "calibration_gpu_ticks": 4939625, "compute_passes": 1, "command_buffer_ns": 4221624.9858, "command_buffers": 2 }, { "compute_ns": 4217750, "compute_span_ns": 4217750, "calibration_cpu_ns": 5011416, "calibration_gpu_ticks": 5011416, "compute_passes": 1, "command_buffer_ns": 4217833.28056, "command_buffers": 2 }, { "compute_ns": 4206792, "compute_span_ns": 4206792, "calibration_cpu_ns": 4843666, "calibration_gpu_ticks": 4843666, "compute_passes": 1, "command_buffer_ns": 4206874.9899, "command_buffers": 2 }, { "compute_ns": 4199542, "compute_span_ns": 4199542, "calibration_cpu_ns": 4841625, "calibration_gpu_ticks": 4841625, "compute_passes": 1, "command_buffer_ns": 4199583.31622, "command_buffers": 2 }, { "compute_ns": 4222958, "compute_span_ns": 4222958, "calibration_cpu_ns": 5015458, "calibration_gpu_ticks": 5015458, "compute_passes": 1, "command_buffer_ns": 4223000.02538, "command_buffers": 2 }, { "compute_ns": 4175041, "compute_span_ns": 4175041, "calibration_cpu_ns": 4946708, "calibration_gpu_ticks": 4946708, "compute_passes": 1, "command_buffer_ns": 4175000.00913, "command_buffers": 2 }, { "compute_ns": 4242333, "compute_span_ns": 4242333, "calibration_cpu_ns": 5129917, "calibration_gpu_ticks": 5129917, "compute_passes": 1, "command_buffer_ns": 4242333.3507, "command_buffers": 2 }, { "compute_ns": 4284750, "compute_span_ns": 4284750, "calibration_cpu_ns": 4882584, "calibration_gpu_ticks": 4882584, "compute_passes": 1, "command_buffer_ns": 4284791.7066, "command_buffers": 2 } ], "latency": [ { "compute_ns": 68792, "compute_span_ns": 68792, "calibration_cpu_ns": 515250, "calibration_gpu_ticks": 515250, "compute_passes": 1, "command_buffer_ns": 68833.2947902, "command_buffers": 2 }, { "compute_ns": 81958, "compute_span_ns": 81958, "calibration_cpu_ns": 685625, "calibration_gpu_ticks": 685625, "compute_passes": 1, "command_buffer_ns": 81958.3656266, "command_buffers": 2 }, { "compute_ns": 65958, "compute_span_ns": 65958, "calibration_cpu_ns": 487792, "calibration_gpu_ticks": 487792, "compute_passes": 1, "command_buffer_ns": 66000.0368953, "command_buffers": 2 }, { "compute_ns": 78792, "compute_span_ns": 78792, "calibration_cpu_ns": 576584, "calibration_gpu_ticks": 576584, "compute_passes": 1, "command_buffer_ns": 78916.7243056, "command_buffers": 2 }, { "compute_ns": 66125, "compute_span_ns": 66125, "calibration_cpu_ns": 495000, "calibration_gpu_ticks": 495000, "compute_passes": 1, "command_buffer_ns": 66124.9505356, "command_buffers": 2 }, { "compute_ns": 74583, "compute_span_ns": 74583, "calibration_cpu_ns": 462542, "calibration_gpu_ticks": 462542, "compute_passes": 1, "command_buffer_ns": 74583.3385736, "command_buffers": 2 }, { "compute_ns": 108292, "compute_span_ns": 108292, "calibration_cpu_ns": 783166, "calibration_gpu_ticks": 783166, "compute_passes": 1, "command_buffer_ns": 108333.304524, "command_buffers": 2 }, { "compute_ns": 67667, "compute_span_ns": 67667, "calibration_cpu_ns": 497834, "calibration_gpu_ticks": 497834, "compute_passes": 1, "command_buffer_ns": 67791.6686982, "command_buffers": 2 }, { "compute_ns": 74583, "compute_span_ns": 74583, "calibration_cpu_ns": 492583, "calibration_gpu_ticks": 492583, "compute_passes": 1, "command_buffer_ns": 74625.0152588, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4218375.01926, "command_buffers": 2 }, { "command_buffer_ns": 4254875.06436, "command_buffers": 2 }, { "command_buffer_ns": 4202749.9876, "command_buffers": 2 }, { "command_buffer_ns": 4225499.98621, "command_buffers": 2 }, { "command_buffer_ns": 4213666.65978, "command_buffers": 2 }, { "command_buffer_ns": 4316375.00878, "command_buffers": 2 }, { "command_buffer_ns": 4203083.34287, "command_buffers": 2 }, { "command_buffer_ns": 4249791.67292, "command_buffers": 2 }, { "command_buffer_ns": 4223916.67962, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 112500.041723, "command_buffers": 2 }, { "command_buffer_ns": 74791.663792, "command_buffers": 2 }, { "command_buffer_ns": 65374.9448247, "command_buffers": 2 }, { "command_buffer_ns": 112916.69216, "command_buffers": 2 }, { "command_buffer_ns": 87083.3173394, "command_buffers": 2 }, { "command_buffer_ns": 67124.9581501, "command_buffers": 2 }, { "command_buffer_ns": 77416.5964685, "command_buffers": 2 }, { "command_buffer_ns": 77000.062447, "command_buffers": 2 }, { "command_buffer_ns": 66625.0707582, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 65.9121096759375, 66.48242288062501, 65.66796855624999, 66.02343728453124, 65.8385415590625, 67.4433595121875, 65.67317723234375, 66.402994889375, 65.9986981190625 ], "command_buffer_throughput_us_p50": 65.9986981190625, "command_buffer_throughput_us_p90": 66.6746102069375, "command_buffer_latency_us": [ 112.500041723, 74.79166379200001, 65.3749448247, 112.91669216000001, 87.0833173394, 67.12495815010001, 77.4165964685, 77.000062447, 66.6250707582 ], "command_buffer_latency_us_p50": 77.000062447, "command_buffer_latency_us_p90": 112.5833718104 }, "compute_throughput_us": [ 65.619796875, 65.96353125, 65.90234375, 65.731125, 65.61784375, 65.98371875, 65.235015625, 66.286453125, 66.94921875 ], "compute_throughput_us_p50": 65.90234375, "compute_throughput_us_p90": 66.41900625, "command_buffer_throughput_us": [ 65.62239559578124, 65.962890403125, 65.90364500875, 65.7324217171875, 65.61848931593751, 65.9843753965625, 65.23437514265625, 66.28645860468751, 66.949870415625 ], "command_buffer_throughput_us_p50": 65.90364500875, "command_buffer_throughput_us_p90": 66.419140966875, "compute_latency_us": [ 68.792, 81.958, 65.958, 78.792, 66.125, 74.583, 108.292, 67.667, 74.583 ], "compute_latency_us_p50": 74.583, "compute_latency_us_p90": 87.2248, "command_buffer_latency_us": [ 68.8332947902, 81.9583656266, 66.0000368953, 78.9167243056, 66.12495053560001, 74.5833385736, 108.33330452400001, 67.7916686982, 74.6250152588 ], "command_buffer_latency_us_p50": 74.5833385736, "command_buffer_latency_us_p90": 87.23335340608, "counter_control_throughput_ratio": 0.9956045394149775, "counter_control_latency_ratio": 1.0095616458610583 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 66.88392, "throughput_us_p90": 68.12862682352, "latency_us_p50": 332.75, "latency_us_p90": 834.7338000000001 }, "slowdown": 0.8026847473072133, "device_compute_slowdown": 0.8452970599635325, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-1lz36snh/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "310a51aaac34ebd04d17becc4645f77793138816428bb43603834b11ab1687fb", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.170167, "capture_ms": 0.058333, "compile_ms": 55.299958, "allocation_upload_ms": 5.730208, "cold_call_ms": 10.17775, "warmup_ms": 200.58575, "download_ms": 2.375792, "repetitions": 457, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 64.8975207877, 66.1202603939, 64.8800131291, 64.9897877462, 65.2636761488, 64.7026805252, 65.0599934354, 65.2401531729, 65.9793019694 ], "latency_us": [ 347.667, 280.042, 279, 277.708, 275.042, 288.125, 291.333, 268.333, 1016.625 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 72, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4108334, "compute_span_ns": 4108334, "calibration_cpu_ns": 5414042, "calibration_gpu_ticks": 5414042, "compute_passes": 1, "command_buffer_ns": 4108458.29174, "command_buffers": 2 }, { "compute_ns": 4110708, "compute_span_ns": 4110708, "calibration_cpu_ns": 5171083, "calibration_gpu_ticks": 5171083, "compute_passes": 1, "command_buffer_ns": 4110708.36708, "command_buffers": 2 }, { "compute_ns": 4087459, "compute_span_ns": 4087459, "calibration_cpu_ns": 4764000, "calibration_gpu_ticks": 4764000, "compute_passes": 1, "command_buffer_ns": 4087541.65983, "command_buffers": 2 }, { "compute_ns": 4092209, "compute_span_ns": 4092209, "calibration_cpu_ns": 4906834, "calibration_gpu_ticks": 4906834, "compute_passes": 1, "command_buffer_ns": 4092291.63779, "command_buffers": 2 }, { "compute_ns": 4068375, "compute_span_ns": 4068375, "calibration_cpu_ns": 4931666, "calibration_gpu_ticks": 4931666, "compute_passes": 1, "command_buffer_ns": 4068375.04124, "command_buffers": 2 }, { "compute_ns": 4073125, "compute_span_ns": 4073125, "calibration_cpu_ns": 4819083, "calibration_gpu_ticks": 4819083, "compute_passes": 1, "command_buffer_ns": 4073166.69589, "command_buffers": 2 }, { "compute_ns": 4105042, "compute_span_ns": 4105042, "calibration_cpu_ns": 6255542, "calibration_gpu_ticks": 6255542, "compute_passes": 1, "command_buffer_ns": 4105083.35335, "command_buffers": 2 }, { "compute_ns": 4107416, "compute_span_ns": 4107416, "calibration_cpu_ns": 4967458, "calibration_gpu_ticks": 4967458, "compute_passes": 1, "command_buffer_ns": 4107374.98896, "command_buffers": 2 }, { "compute_ns": 4076583, "compute_span_ns": 4076583, "calibration_cpu_ns": 4741792, "calibration_gpu_ticks": 4741792, "compute_passes": 1, "command_buffer_ns": 4076583.31096, "command_buffers": 2 } ], "latency": [ { "compute_ns": 90458, "compute_span_ns": 90458, "calibration_cpu_ns": 618250, "calibration_gpu_ticks": 618250, "compute_passes": 1, "command_buffer_ns": 90500.0488274, "command_buffers": 2 }, { "compute_ns": 84875, "compute_span_ns": 84875, "calibration_cpu_ns": 774250, "calibration_gpu_ticks": 774250, "compute_passes": 1, "command_buffer_ns": 84999.94874, "command_buffers": 2 }, { "compute_ns": 77292, "compute_span_ns": 77292, "calibration_cpu_ns": 494125, "calibration_gpu_ticks": 494125, "compute_passes": 1, "command_buffer_ns": 77375.0361986, "command_buffers": 2 }, { "compute_ns": 107042, "compute_span_ns": 107042, "calibration_cpu_ns": 769416, "calibration_gpu_ticks": 769416, "compute_passes": 1, "command_buffer_ns": 107166.648377, "command_buffers": 2 }, { "compute_ns": 64959, "compute_span_ns": 64959, "calibration_cpu_ns": 503167, "calibration_gpu_ticks": 503167, "compute_passes": 1, "command_buffer_ns": 65083.3244435, "command_buffers": 2 }, { "compute_ns": 64666, "compute_span_ns": 64666, "calibration_cpu_ns": 446250, "calibration_gpu_ticks": 446250, "compute_passes": 1, "command_buffer_ns": 64666.6740067, "command_buffers": 2 }, { "compute_ns": 98583, "compute_span_ns": 98583, "calibration_cpu_ns": 752750, "calibration_gpu_ticks": 752750, "compute_passes": 1, "command_buffer_ns": 98625.0233836, "command_buffers": 2 }, { "compute_ns": 66375, "compute_span_ns": 66375, "calibration_cpu_ns": 558333, "calibration_gpu_ticks": 558333, "compute_passes": 1, "command_buffer_ns": 66458.3640173, "command_buffers": 2 }, { "compute_ns": 64917, "compute_span_ns": 64917, "calibration_cpu_ns": 465667, "calibration_gpu_ticks": 465667, "compute_passes": 1, "command_buffer_ns": 64958.3525956, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4099458.33962, "command_buffers": 2 }, { "command_buffer_ns": 4066791.67645, "command_buffers": 2 }, { "command_buffer_ns": 4110083.39142, "command_buffers": 2 }, { "command_buffer_ns": 4080666.63658, "command_buffers": 2 }, { "command_buffer_ns": 4084458.34182, "command_buffers": 2 }, { "command_buffer_ns": 4103166.63329, "command_buffers": 2 }, { "command_buffer_ns": 4095333.39553, "command_buffers": 2 }, { "command_buffer_ns": 4110749.92735, "command_buffers": 2 }, { "command_buffer_ns": 4068583.36646, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 97458.367236, "command_buffers": 2 }, { "command_buffer_ns": 64958.2943879, "command_buffers": 2 }, { "command_buffer_ns": 65083.3826512, "command_buffers": 2 }, { "command_buffer_ns": 73958.246503, "command_buffers": 2 }, { "command_buffer_ns": 87124.9940246, "command_buffers": 2 }, { "command_buffer_ns": 67833.3453834, "command_buffers": 2 }, { "command_buffer_ns": 77583.3614171, "command_buffers": 2 }, { "command_buffer_ns": 65208.3544992, "command_buffers": 2 }, { "command_buffer_ns": 65708.3583064, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 64.05403655656251, 63.54361994453125, 64.2200529909375, 63.7604161965625, 63.8196615909375, 64.11197864515624, 63.98958430515625, 64.23046761484375, 63.5716151009375 ], "command_buffer_throughput_us_p50": 63.98958430515625, "command_buffer_throughput_us_p90": 64.22213591571875, "command_buffer_latency_us": [ 97.458367236, 64.9582943879, 65.0833826512, 73.958246503, 87.1249940246, 67.8333453834, 77.5833614171, 65.2083544992, 65.7083583064 ], "command_buffer_latency_us_p50": 67.8333453834, "command_buffer_latency_us_p90": 89.19166866688 }, "compute_throughput_us": [ 64.19271875, 64.2298125, 63.866546875, 63.940765625, 63.568359375, 63.642578125, 64.14128125, 64.178375, 63.696609375 ], "compute_throughput_us_p50": 63.940765625, "compute_throughput_us_p90": 64.2001375, "command_buffer_throughput_us": [ 64.19466080843749, 64.229818235625, 63.86783843484375, 63.94205684046875, 63.568360019375, 63.64322962328125, 64.14192739609375, 64.1777342025, 63.69661423375 ], "command_buffer_throughput_us_p50": 63.94205684046875, "command_buffer_throughput_us_p90": 64.201692293875, "compute_latency_us": [ 90.458, 84.875, 77.292, 107.042, 64.959, 64.666, 98.583, 66.375, 64.917 ], "compute_latency_us_p50": 77.292, "compute_latency_us_p90": 100.2748, "command_buffer_latency_us": [ 90.5000488274, 84.99994874000001, 77.3750361986, 107.166648377, 65.0833244435, 64.66667400669999, 98.6250233836, 66.4583640173, 64.9583525956 ], "command_buffer_latency_us_p50": 77.3750361986, "command_buffer_latency_us_p90": 100.33334838228, "counter_control_throughput_ratio": 1.0019662727242973, "counter_control_latency_ratio": 1.0191694688157686 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 65.0599934354, "throughput_us_p90": 66.00749365429999, "latency_us_p50": 280.042, "latency_us_p90": 481.4586000000001 }, "torch": { "cold_call_ms": 7.482667, "warmup_ms": 200.356166, "repetitions": 377, "throughput_us": [ 72.30349071618038, 72.82681432360742, 74.63063660477454, 72.16810344827586, 72.64666047745358, 73.01215649867375, 72.41246684350132, 72.26204774535809, 72.85046419098143 ], "latency_us": [ 414.417, 364.20799999999997, 439.125, 396.25, 322.75, 310.58299999999997, 331.083, 294.917, 327.41700000000003 ], "throughput_us_p50": 72.64666047745358, "throughput_us_p90": 73.3358525198939, "latency_us_p50": 331.083, "latency_us_p90": 419.35859999999997, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4495500.004850328, "command_buffers": 1 }, { "command_buffer_ns": 4420125.04208833, "command_buffers": 1 }, { "command_buffer_ns": 4422666.679602116, "command_buffers": 1 }, { "command_buffer_ns": 4422375.001013279, "command_buffers": 1 }, { "command_buffer_ns": 4399208.293762058, "command_buffers": 1 }, { "command_buffer_ns": 4410000.052303076, "command_buffers": 1 }, { "command_buffer_ns": 4401958.372909576, "command_buffers": 1 }, { "command_buffer_ns": 4422375.001013279, "command_buffers": 1 }, { "command_buffer_ns": 4418625.03066659, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 104416.6274368763, "command_buffers": 1 }, { "command_buffer_ns": 71374.99051168561, "command_buffers": 1 }, { "command_buffer_ns": 70458.33626762033, "command_buffers": 1 }, { "command_buffer_ns": 94958.34819972515, "command_buffers": 1 }, { "command_buffer_ns": 83499.99552592635, "command_buffers": 1 }, { "command_buffer_ns": 70333.36441963911, "command_buffers": 1 }, { "command_buffer_ns": 70041.68583080173, "command_buffers": 1 }, { "command_buffer_ns": 76249.99852851033, "command_buffers": 1 }, { "command_buffer_ns": 71874.9943189323, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 70.24218757578637, 69.06445378263015, 69.10416686878307, 69.09960939083248, 68.73762959003216, 68.90625081723556, 68.78059957671212, 69.09960939083248, 69.04101610416546 ], "command_buffer_throughput_us_p50": 69.06445378263015, "command_buffer_throughput_us_p90": 69.33177101018373, "command_buffer_latency_us": [ 104.4166274368763, 71.37499051168561, 70.45833626762033, 94.95834819972515, 83.49999552592635, 70.33336441963911, 70.04168583080173, 76.24999852851033, 71.8749943189323 ], "command_buffer_latency_us_p50": 71.8749943189323, "command_buffer_latency_us_p90": 96.85000404715538 }, "throughput": [ { "compute_ns": 4308125.0, "compute_span_ns": 4308125.0, "command_buffer_ns": 4308125.004172325, "calibration_cpu_ns": 5326584.0, "calibration_gpu_ticks": 5326584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4411833.0, "compute_span_ns": 4411833.0, "command_buffer_ns": 4411791.684105992, "calibration_cpu_ns": 5267125.0, "calibration_gpu_ticks": 5267125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4426666.0, "compute_span_ns": 4426666.0, "command_buffer_ns": 4426625.033374876, "calibration_cpu_ns": 5276500.0, "calibration_gpu_ticks": 5276500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4461209.0, "compute_span_ns": 4461209.0, "command_buffer_ns": 4461249.976884574, "calibration_cpu_ns": 5334459.0, "calibration_gpu_ticks": 5334459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4452500.0, "compute_span_ns": 4452500.0, "command_buffer_ns": 4452500.026673079, "calibration_cpu_ns": 5384375.0, "calibration_gpu_ticks": 5384375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4401583.0, "compute_span_ns": 4401583.0, "command_buffer_ns": 4401541.664265096, "calibration_cpu_ns": 5251625.0, "calibration_gpu_ticks": 5251625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4460417.0, "compute_span_ns": 4460417.0, "command_buffer_ns": 4460458.352696151, "calibration_cpu_ns": 5362125.0, "calibration_gpu_ticks": 5362125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4397209.0, "compute_span_ns": 4397209.0, "command_buffer_ns": 4397250.013425946, "calibration_cpu_ns": 5353167.0, "calibration_gpu_ticks": 5353167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4452083.0, "compute_span_ns": 4452083.0, "command_buffer_ns": 4452083.318028599, "calibration_cpu_ns": 5486667.0, "calibration_gpu_ticks": 5486667.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 71583.0, "compute_span_ns": 71583.0, "command_buffer_ns": 71541.69725254178, "calibration_cpu_ns": 658333.0, "calibration_gpu_ticks": 658333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71792.0, "compute_span_ns": 71792.0, "command_buffer_ns": 71833.37584137917, "calibration_cpu_ns": 595750.0, "calibration_gpu_ticks": 595750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70625.0, "compute_span_ns": 70625.0, "command_buffer_ns": 70625.0430084765, "calibration_cpu_ns": 520916.0, "calibration_gpu_ticks": 520916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71083.0, "compute_span_ns": 71083.0, "command_buffer_ns": 71083.37013050914, "calibration_cpu_ns": 608875.0, "calibration_gpu_ticks": 608875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72500.0, "compute_span_ns": 72500.0, "command_buffer_ns": 72499.9699741602, "calibration_cpu_ns": 606833.0, "calibration_gpu_ticks": 606833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72709.0, "compute_span_ns": 72709.0, "command_buffer_ns": 72749.97187778354, "calibration_cpu_ns": 534583.0, "calibration_gpu_ticks": 534583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 105000.0, "compute_span_ns": 105000.0, "command_buffer_ns": 104999.98461455107, "calibration_cpu_ns": 793375.0, "calibration_gpu_ticks": 793375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82542.0, "compute_span_ns": 82542.0, "command_buffer_ns": 82583.34128186107, "calibration_cpu_ns": 543625.0, "calibration_gpu_ticks": 543625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70125.0, "compute_span_ns": 70125.0, "command_buffer_ns": 70124.9809935689, "calibration_cpu_ns": 568708.0, "calibration_gpu_ticks": 568708.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 67.314453125, 68.934890625, 69.16665625, 69.706390625, 69.5703125, 68.774734375, 69.694015625, 68.706390625, 69.563796875 ], "compute_throughput_us_p50": 69.16665625, "compute_throughput_us_p90": 69.69649062500001, "command_buffer_throughput_us": [ 67.31445319019258, 68.93424506415613, 69.16601614648243, 69.70703088882146, 69.57031291676685, 68.77408850414213, 69.69466176087735, 68.70703145978041, 69.56380184419686 ], "command_buffer_throughput_us_p50": 69.16601614648243, "command_buffer_throughput_us_p90": 69.69713558646617, "compute_latency_us": [ 71.583, 71.792, 70.625, 71.083, 72.5, 72.709, 105.0, 82.542, 70.125 ], "compute_latency_us_p50": 71.792, "compute_latency_us_p90": 87.0336, "command_buffer_latency_us": [ 71.54169725254178, 71.83337584137917, 70.6250430084765, 71.08337013050914, 72.4999699741602, 72.74997187778354, 104.99998461455107, 82.58334128186107, 70.1249809935689 ], "command_buffer_latency_us_p50": 71.83337584137917, "command_buffer_latency_us_p90": 87.06666994839907, "counter_control_throughput_ratio": 1.000895015170602, "counter_control_latency_ratio": 1.0023660328881876 }, "download_ms": 0.634375, "allocation_upload_ms": 16.872291, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.8955675733448456, "device_compute_slowdown": 0.9244449434405034, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.382291, "warmup_ms": 200.756916, "repetitions": 393, "throughput_us": [ 72.55290585241731, 72.91963613231552, 72.26378117048347, 73.72731043256996, 72.94582188295166, 72.33449872773538, 72.87606106870228, 72.54675572519083, 74.5637175572519 ], "latency_us": [ 412.209, 305.25, 297.292, 309.79200000000003, 287.292, 369.542, 290.416, 287.0, 298.125 ], "throughput_us_p50": 72.87606106870228, "throughput_us_p90": 73.89459185750636, "latency_us_p50": 298.125, "latency_us_p90": 378.0754, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4524791.671428829, "command_buffers": 1 }, { "command_buffer_ns": 4486749.996431172, "command_buffers": 1 }, { "command_buffer_ns": 4594250.000081956, "command_buffers": 1 }, { "command_buffer_ns": 4409499.990288168, "command_buffers": 1 }, { "command_buffer_ns": 4488624.9816045165, "command_buffers": 1 }, { "command_buffer_ns": 4481124.982703477, "command_buffers": 1 }, { "command_buffer_ns": 4490333.376452327, "command_buffers": 1 }, { "command_buffer_ns": 4473249.951843172, "command_buffers": 1 }, { "command_buffer_ns": 4471874.970477074, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 97750.04582479596, "command_buffers": 1 }, { "command_buffer_ns": 69874.97908994555, "command_buffers": 1 }, { "command_buffer_ns": 70041.62762314081, "command_buffers": 1 }, { "command_buffer_ns": 84458.32645520568, "command_buffers": 1 }, { "command_buffer_ns": 90166.6353456676, "command_buffers": 1 }, { "command_buffer_ns": 71749.96426329017, "command_buffers": 1 }, { "command_buffer_ns": 70541.6314303875, "command_buffers": 1 }, { "command_buffer_ns": 71166.66529327631, "command_buffers": 1 }, { "command_buffer_ns": 86041.6330397129, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 70.69986986607546, 70.10546869423706, 71.78515625128057, 68.89843734825263, 70.13476533757057, 70.01757785474183, 70.1614590070676, 69.89453049754957, 69.87304641370429 ], "command_buffer_throughput_us_p50": 70.10546869423706, "command_buffer_throughput_us_p90": 70.91692714311648, "command_buffer_latency_us": [ 97.75004582479596, 69.87497908994555, 70.04162762314081, 84.45832645520568, 90.1666353456676, 71.74996426329017, 70.5416314303875, 71.16666529327631, 86.0416330397129 ], "command_buffer_latency_us_p50": 71.74996426329017, "command_buffer_latency_us_p90": 91.68331744149327 }, "throughput": [ { "compute_ns": 4451458.0, "compute_span_ns": 4451458.0, "command_buffer_ns": 4451416.665688157, "calibration_cpu_ns": 5308083.0, "calibration_gpu_ticks": 5308083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4455375.0, "compute_span_ns": 4455375.0, "command_buffer_ns": 4455375.0194609165, "calibration_cpu_ns": 6089042.0, "calibration_gpu_ticks": 6089042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4479458.0, "compute_span_ns": 4479458.0, "command_buffer_ns": 4479416.646063328, "calibration_cpu_ns": 5339583.0, "calibration_gpu_ticks": 5339583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4468042.0, "compute_span_ns": 4468042.0, "command_buffer_ns": 4468041.704967618, "calibration_cpu_ns": 5324417.0, "calibration_gpu_ticks": 5324417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4488000.0, "compute_span_ns": 4488000.0, "command_buffer_ns": 4488000.005949289, "calibration_cpu_ns": 5326083.0, "calibration_gpu_ticks": 5326083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4496667.0, "compute_span_ns": 4496667.0, "command_buffer_ns": 4496666.660998017, "calibration_cpu_ns": 5343125.0, "calibration_gpu_ticks": 5343125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4425791.0, "compute_span_ns": 4425791.0, "command_buffer_ns": 4425749.997608364, "calibration_cpu_ns": 5570583.0, "calibration_gpu_ticks": 5570583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4440333.0, "compute_span_ns": 4440333.0, "command_buffer_ns": 4440291.66828841, "calibration_cpu_ns": 5290291.0, "calibration_gpu_ticks": 5290291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4498250.0, "compute_span_ns": 4498250.0, "command_buffer_ns": 4498249.967582524, "calibration_cpu_ns": 5505750.0, "calibration_gpu_ticks": 5505750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 83917.0, "compute_span_ns": 83917.0, "command_buffer_ns": 83916.64596274495, "calibration_cpu_ns": 558667.0, "calibration_gpu_ticks": 558667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 92375.0, "compute_span_ns": 92375.0, "command_buffer_ns": 92375.03400072455, "calibration_cpu_ns": 902542.0, "calibration_gpu_ticks": 902542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71125.0, "compute_span_ns": 71125.0, "command_buffer_ns": 71125.04681572318, "calibration_cpu_ns": 529584.0, "calibration_gpu_ticks": 529584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104708.0, "compute_span_ns": 104708.0, "command_buffer_ns": 104708.3642333746, "calibration_cpu_ns": 725417.0, "calibration_gpu_ticks": 725417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70792.0, "compute_span_ns": 70792.0, "command_buffer_ns": 70791.63333401084, "calibration_cpu_ns": 483708.0, "calibration_gpu_ticks": 483708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70084.0, "compute_span_ns": 70084.0, "command_buffer_ns": 70125.03920122981, "calibration_cpu_ns": 487000.0, "calibration_gpu_ticks": 487000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 96750.0, "compute_span_ns": 96750.0, "command_buffer_ns": 96749.98000264168, "calibration_cpu_ns": 709500.0, "calibration_gpu_ticks": 709500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70791.0, "compute_span_ns": 70791.0, "command_buffer_ns": 70750.01485645771, "calibration_cpu_ns": 508875.0, "calibration_gpu_ticks": 508875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71583.0, "compute_span_ns": 71583.0, "command_buffer_ns": 71541.63904488087, "calibration_cpu_ns": 501166.0, "calibration_gpu_ticks": 501166.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 69.55403125, 69.615234375, 69.99153125, 69.81315625, 70.125, 70.260421875, 69.152984375, 69.380203125, 70.28515625 ], "compute_throughput_us_p50": 69.81315625, "compute_throughput_us_p90": 70.26536875000001, "command_buffer_throughput_us": [ 69.55338540137745, 69.61523467907682, 69.9908850947395, 69.81315164011903, 70.12500009295763, 70.26041657809401, 69.15234371263068, 69.3795573170064, 70.28515574347693 ], "command_buffer_throughput_us_p50": 69.81315164011903, "command_buffer_throughput_us_p90": 70.2653644111706, "compute_latency_us": [ 83.917, 92.375, 71.125, 104.708, 70.792, 70.084, 96.75, 70.791, 71.583 ], "compute_latency_us_p50": 71.583, "compute_latency_us_p90": 98.3416, "command_buffer_latency_us": [ 83.91664596274495, 92.37503400072455, 71.12504681572318, 104.7083642333746, 70.79163333401084, 70.12503920122981, 96.74998000264168, 70.75001485645771, 71.54163904488087 ], "command_buffer_latency_us_p50": 71.54163904488087, "command_buffer_latency_us_p90": 98.34165684878826, "counter_control_throughput_ratio": 0.9930071929581075, "counter_control_latency_ratio": 0.9941454270043202 }, "download_ms": 0.630916, "allocation_upload_ms": 1.213708, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-aho14kb1/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "26111b6b600e52c50494ad3f783ef6fc3b02252d329d47ea39eb66749773d18e", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 52.696167, "capture_ms": 0.070583, "compile_ms": 79.577791, "allocation_upload_ms": 6.842333, "cold_call_ms": 6.029417, "warmup_ms": 200.879791, "download_ms": 3.095959, "repetitions": 503, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 53.3310139165, 53.4697654076, 53.4648787276, 53.4254473161, 54.0708250497, 54.4107017893, 53.7355029821, 53.7854532803, 53.2118946322 ], "latency_us": [ 361.208, 322.459, 692.667, 289.166, 318.958, 387.875, 320.167, 398.541, 335.334 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 1280, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3368458, "compute_span_ns": 3368458, "calibration_cpu_ns": 3862084, "calibration_gpu_ticks": 3862084, "compute_passes": 1, "command_buffer_ns": 3368458.36136, "command_buffers": 2 }, { "compute_ns": 3476959, "compute_span_ns": 3476959, "calibration_cpu_ns": 4149000, "calibration_gpu_ticks": 4149000, "compute_passes": 1, "command_buffer_ns": 3477041.66779, "command_buffers": 2 }, { "compute_ns": 3323750, "compute_span_ns": 3323750, "calibration_cpu_ns": 3983042, "calibration_gpu_ticks": 3983042, "compute_passes": 1, "command_buffer_ns": 3323791.66503, "command_buffers": 2 }, { "compute_ns": 3443375, "compute_span_ns": 3443375, "calibration_cpu_ns": 3950541, "calibration_gpu_ticks": 3950541, "compute_passes": 1, "command_buffer_ns": 3443374.997, "command_buffers": 2 }, { "compute_ns": 3371459, "compute_span_ns": 3371459, "calibration_cpu_ns": 4536875, "calibration_gpu_ticks": 4536875, "compute_passes": 1, "command_buffer_ns": 3371500.00269, "command_buffers": 2 }, { "compute_ns": 3352500, "compute_span_ns": 3352500, "calibration_cpu_ns": 4175959, "calibration_gpu_ticks": 4175959, "compute_passes": 1, "command_buffer_ns": 3352541.5929, "command_buffers": 2 }, { "compute_ns": 3363333, "compute_span_ns": 3363333, "calibration_cpu_ns": 4146666, "calibration_gpu_ticks": 4146666, "compute_passes": 1, "command_buffer_ns": 3363374.96992, "command_buffers": 2 }, { "compute_ns": 3460292, "compute_span_ns": 3460292, "calibration_cpu_ns": 4145208, "calibration_gpu_ticks": 4145208, "compute_passes": 1, "command_buffer_ns": 3460416.68672, "command_buffers": 2 }, { "compute_ns": 3484250, "compute_span_ns": 3484250, "calibration_cpu_ns": 4424458, "calibration_gpu_ticks": 4424458, "compute_passes": 1, "command_buffer_ns": 3484333.34148, "command_buffers": 2 } ], "latency": [ { "compute_ns": 83666, "compute_span_ns": 83666, "calibration_cpu_ns": 726667, "calibration_gpu_ticks": 726667, "compute_passes": 1, "command_buffer_ns": 83666.7022668, "command_buffers": 2 }, { "compute_ns": 54250, "compute_span_ns": 54250, "calibration_cpu_ns": 460375, "calibration_gpu_ticks": 460375, "compute_passes": 1, "command_buffer_ns": 54333.3007954, "command_buffers": 2 }, { "compute_ns": 54291, "compute_span_ns": 54291, "calibration_cpu_ns": 497333, "calibration_gpu_ticks": 497333, "compute_passes": 1, "command_buffer_ns": 54250.0056326, "command_buffers": 2 }, { "compute_ns": 91250, "compute_span_ns": 91250, "calibration_cpu_ns": 722833, "calibration_gpu_ticks": 722833, "compute_passes": 1, "command_buffer_ns": 91291.6730158, "command_buffers": 2 }, { "compute_ns": 66000, "compute_span_ns": 66000, "calibration_cpu_ns": 530833, "calibration_gpu_ticks": 530833, "compute_passes": 1, "command_buffer_ns": 66083.2738504, "command_buffers": 2 }, { "compute_ns": 54458, "compute_span_ns": 54458, "calibration_cpu_ns": 456958, "calibration_gpu_ticks": 456958, "compute_passes": 1, "command_buffer_ns": 54458.330851, "command_buffers": 2 }, { "compute_ns": 88500, "compute_span_ns": 88500, "calibration_cpu_ns": 666208, "calibration_gpu_ticks": 666208, "compute_passes": 1, "command_buffer_ns": 88583.3287612, "command_buffers": 2 }, { "compute_ns": 65375, "compute_span_ns": 65375, "calibration_cpu_ns": 858458, "calibration_gpu_ticks": 858458, "compute_passes": 1, "command_buffer_ns": 65375.0030324, "command_buffers": 2 }, { "compute_ns": 54500, "compute_span_ns": 54500, "calibration_cpu_ns": 478416, "calibration_gpu_ticks": 478416, "compute_passes": 1, "command_buffer_ns": 54624.9793842, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3498833.33547, "command_buffers": 2 }, { "command_buffer_ns": 3444250.03277, "command_buffers": 2 }, { "command_buffer_ns": 3437874.95513, "command_buffers": 2 }, { "command_buffer_ns": 3346708.31216, "command_buffers": 2 }, { "command_buffer_ns": 3407916.578, "command_buffers": 2 }, { "command_buffer_ns": 3412500.02384, "command_buffers": 2 }, { "command_buffer_ns": 3323375.01459, "command_buffers": 2 }, { "command_buffer_ns": 3421916.6846, "command_buffers": 2 }, { "command_buffer_ns": 3342916.72334, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 98083.4010988, "command_buffers": 2 }, { "command_buffer_ns": 54625.0375919, "command_buffers": 2 }, { "command_buffer_ns": 55708.2821615, "command_buffers": 2 }, { "command_buffer_ns": 53958.3852515, "command_buffers": 2 }, { "command_buffer_ns": 67541.6667946, "command_buffers": 2 }, { "command_buffer_ns": 54666.6560695, "command_buffers": 2 }, { "command_buffer_ns": 54874.9812879, "command_buffers": 2 }, { "command_buffer_ns": 79833.320342, "command_buffers": 2 }, { "command_buffer_ns": 53500.0581294, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 54.66927086671875, 53.81640676203125, 53.71679617390625, 52.292317377500005, 53.248696531250005, 53.3203128725, 51.92773460296875, 53.467448196875, 52.2330738021875 ], "command_buffer_throughput_us_p50": 53.3203128725, "command_buffer_throughput_us_p90": 53.98697958296875, "command_buffer_latency_us": [ 98.0834010988, 54.625037591899996, 55.7082821615, 53.958385251500005, 67.54166679459999, 54.6666560695, 54.8749812879, 79.83332034200001, 53.5000581294 ], "command_buffer_latency_us_p50": 54.8749812879, "command_buffer_latency_us_p90": 83.48333649336001 }, "compute_throughput_us": [ 52.63215625, 54.327484375, 51.93359375, 53.802734375, 52.679046875, 52.3828125, 52.552078125, 54.0670625, 54.44140625 ], "compute_throughput_us_p50": 52.679046875, "compute_throughput_us_p90": 54.35026875, "command_buffer_throughput_us": [ 52.63216189625, 54.32877605921875, 51.93424476609375, 53.802734328125, 52.67968754203125, 52.3834623890625, 52.552733905000004, 54.06901073, 54.442708460625 ], "command_buffer_throughput_us_p50": 52.67968754203125, "command_buffer_throughput_us_p90": 54.351562539499994, "compute_latency_us": [ 83.666, 54.25, 54.291, 91.25, 66.0, 54.458, 88.5, 65.375, 54.5 ], "compute_latency_us_p50": 65.375, "compute_latency_us_p90": 89.05, "command_buffer_latency_us": [ 83.6667022668, 54.3333007954, 54.2500056326, 91.2916730158, 66.0832738504, 54.458330851, 88.5833287612, 65.3750030324, 54.6249793842 ], "command_buffer_latency_us_p50": 65.3750030324, "command_buffer_latency_us_p90": 89.12499761212, "counter_control_throughput_ratio": 1.0095206894702786, "counter_control_latency_ratio": 0.9946592842886526 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 53.4697654076, "throughput_us_p90": 54.138800397619995, "latency_us_p50": 335.334, "latency_us_p90": 457.36620000000005 }, "slowdown": 0.7337082249436144, "device_compute_slowdown": 0.7545719131556954, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-2b9qrg_v/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "90a0b6dbe0fbfeffc09b3a8db9fc5856f9e2631dc2ce0e6d9ee991918165a6cf", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.603958, "capture_ms": 0.078875, "compile_ms": 63.395792, "allocation_upload_ms": 1.286833, "cold_call_ms": 2.052083, "warmup_ms": 200.057, "download_ms": 0.288042, "repetitions": 5733, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.59467852782, 4.49701290773, 4.55004639805, 4.5567402756, 4.48808791209, 4.80329530787, 4.51412140241, 4.46876992848, 4.51284964242 ], "latency_us": [ 265.541, 206.083, 217.25, 238.125, 248.25, 221.541, 214.208, 972.667, 222.584 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 281709, "compute_span_ns": 281709, "calibration_cpu_ns": 734084, "calibration_gpu_ticks": 734084, "compute_passes": 1, "command_buffer_ns": 281749.9917, "command_buffers": 2 }, { "compute_ns": 304250, "compute_span_ns": 304250, "calibration_cpu_ns": 868083, "calibration_gpu_ticks": 868083, "compute_passes": 1, "command_buffer_ns": 304374.960251, "command_buffers": 2 }, { "compute_ns": 284791, "compute_span_ns": 284791, "calibration_cpu_ns": 850708, "calibration_gpu_ticks": 850708, "compute_passes": 1, "command_buffer_ns": 284833.309706, "command_buffers": 2 }, { "compute_ns": 262292, "compute_span_ns": 262292, "calibration_cpu_ns": 757000, "calibration_gpu_ticks": 757000, "compute_passes": 1, "command_buffer_ns": 262416.724581, "command_buffers": 2 }, { "compute_ns": 263625, "compute_span_ns": 263625, "calibration_cpu_ns": 732958, "calibration_gpu_ticks": 732958, "compute_passes": 1, "command_buffer_ns": 263708.294369, "command_buffers": 2 }, { "compute_ns": 282833, "compute_span_ns": 282833, "calibration_cpu_ns": 680666, "calibration_gpu_ticks": 680666, "compute_passes": 1, "command_buffer_ns": 282791.676, "command_buffers": 2 }, { "compute_ns": 310042, "compute_span_ns": 310042, "calibration_cpu_ns": 846416, "calibration_gpu_ticks": 846416, "compute_passes": 1, "command_buffer_ns": 310125.004034, "command_buffers": 2 }, { "compute_ns": 264334, "compute_span_ns": 264334, "calibration_cpu_ns": 717584, "calibration_gpu_ticks": 717584, "compute_passes": 1, "command_buffer_ns": 264416.681603, "command_buffers": 2 }, { "compute_ns": 282541, "compute_span_ns": 282541, "calibration_cpu_ns": 964875, "calibration_gpu_ticks": 964875, "compute_passes": 1, "command_buffer_ns": 282583.292574, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 468000, "calibration_gpu_ticks": 468000, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 7542, "compute_span_ns": 7542, "calibration_cpu_ns": 394917, "calibration_gpu_ticks": 394917, "compute_passes": 1, "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 416667, "calibration_gpu_ticks": 416667, "compute_passes": 1, "command_buffer_ns": 7291.61547497, "command_buffers": 2 }, { "compute_ns": 10375, "compute_span_ns": 10375, "calibration_cpu_ns": 787208, "calibration_gpu_ticks": 787208, "compute_passes": 1, "command_buffer_ns": 10499.9635369, "command_buffers": 2 }, { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 403875, "calibration_gpu_ticks": 403875, "compute_passes": 1, "command_buffer_ns": 7333.29216018, "command_buffers": 2 }, { "compute_ns": 10500, "compute_span_ns": 10500, "calibration_cpu_ns": 357708, "calibration_gpu_ticks": 357708, "compute_passes": 1, "command_buffer_ns": 10625.0518002, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 440500, "calibration_gpu_ticks": 440500, "compute_passes": 1, "command_buffer_ns": 7208.37851986, "command_buffers": 2 }, { "compute_ns": 8416, "compute_span_ns": 8416, "calibration_cpu_ns": 833125, "calibration_gpu_ticks": 833125, "compute_passes": 1, "command_buffer_ns": 8458.27162266, "command_buffers": 2 }, { "compute_ns": 7333, "compute_span_ns": 7333, "calibration_cpu_ns": 384584, "calibration_gpu_ticks": 384584, "compute_passes": 1, "command_buffer_ns": 7375.02705306, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 309625.000227, "command_buffers": 2 }, { "command_buffer_ns": 285583.373625, "command_buffers": 2 }, { "command_buffer_ns": 262541.638222, "command_buffers": 2 }, { "command_buffer_ns": 285166.66498, "command_buffers": 2 }, { "command_buffer_ns": 298791.681416, "command_buffers": 2 }, { "command_buffer_ns": 264916.627202, "command_buffers": 2 }, { "command_buffer_ns": 284708.337858, "command_buffers": 2 }, { "command_buffer_ns": 280125.00843, "command_buffers": 2 }, { "command_buffer_ns": 262833.31681, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "command_buffer_ns": 7083.29025656, "command_buffers": 2 }, { "command_buffer_ns": 7375.02705306, "command_buffers": 2 }, { "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "command_buffer_ns": 11666.6778922, "command_buffers": 2 }, { "command_buffer_ns": 7208.26210454, "command_buffers": 2 }, { "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "command_buffer_ns": 7125.02514943, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.837890628546875, 4.462240212890626, 4.102213097218749, 4.4557291403125, 4.668620022125, 4.1393223000312505, 4.44856777903125, 4.37695325671875, 4.10677057515625 ], "command_buffer_throughput_us_p50": 4.44856777903125, "command_buffer_throughput_us_p90": 4.7024741434093755, "command_buffer_latency_us": [ 7.66664743423, 7.08329025656, 7.37502705306, 7.0416717790100005, 7.54167558625, 11.6666778922, 7.20826210454, 7.1667018346499995, 7.12502514943 ], "command_buffer_latency_us_p50": 7.20826210454, "command_buffer_latency_us_p90": 8.466653525824 }, "compute_throughput_us": [ 4.401703125, 4.75390625, 4.449859375, 4.0983125, 4.119140625, 4.419265625, 4.84440625, 4.13021875, 4.414703125 ], "compute_throughput_us_p50": 4.414703125, "compute_throughput_us_p90": 4.77200625, "command_buffer_throughput_us": [ 4.4023436203125, 4.755858753921875, 4.4505204641562495, 4.100261321578125, 4.120442099515625, 4.4186199375, 4.8457031880312496, 4.131510650046875, 4.415363946468751 ], "command_buffer_throughput_us_p50": 4.415363946468751, "command_buffer_throughput_us_p90": 4.77382764074375, "compute_latency_us": [ 7.042, 7.542, 7.25, 10.375, 7.25, 10.5, 7.125, 8.416, 7.333 ], "compute_latency_us_p50": 7.333, "compute_latency_us_p90": 10.4, "command_buffer_latency_us": [ 7.208320312200001, 7.54167558625, 7.2916154749699995, 10.499963536900001, 7.33329216018, 10.625051800200001, 7.20837851986, 8.45827162266, 7.37502705306 ], "command_buffer_latency_us_p50": 7.37502705306, "command_buffer_latency_us_p90": 10.524981189560002, "counter_control_throughput_ratio": 1.0658007025670733, "counter_control_latency_ratio": 1.0000161502617846 }, "correctness": { "max_abs_error": 2.43199801630567e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 4.51412140241, "throughput_us_p90": 4.6364018838300005, "latency_us_p50": 222.584, "latency_us_p90": 406.96620000000013 }, "torch": { "cold_call_ms": 9.819, "warmup_ms": 200.125458, "repetitions": 2385, "throughput_us": [ 12.548934171907757, 12.025576519916143, 12.49112536687631, 12.013259958071279, 12.010656603773585, 11.802795387840671, 11.88539496855346, 11.686705241090147, 11.790129140461216 ], "latency_us": [ 401.375, 273.625, 216.084, 209.625, 565.75, 494.833, 234.042, 230.584, 394.375 ], "throughput_us_p50": 12.010656603773585, "throughput_us_p90": 12.502687127882599, "latency_us_p50": 273.625, "latency_us_p90": 509.01640000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 486250.03546476364, "command_buffers": 1 }, { "command_buffer_ns": 486791.65774956346, "command_buffers": 1 }, { "command_buffer_ns": 497208.32612365484, "command_buffers": 1 }, { "command_buffer_ns": 499000.0161342323, "command_buffers": 1 }, { "command_buffer_ns": 485000.0259466469, "command_buffers": 1 }, { "command_buffer_ns": 518625.0200495124, "command_buffers": 1 }, { "command_buffer_ns": 506041.687913239, "command_buffers": 1 }, { "command_buffer_ns": 487499.98677521944, "command_buffers": 1 }, { "command_buffer_ns": 566000.0024363399, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 11666.6778922081, "command_buffers": 1 }, { "command_buffer_ns": 10499.963536858559, "command_buffers": 1 }, { "command_buffer_ns": 10750.023648142815, "command_buffers": 1 }, { "command_buffer_ns": 10500.021744519472, "command_buffers": 1 }, { "command_buffer_ns": 10458.34505930543, "command_buffers": 1 }, { "command_buffer_ns": 10375.049896538258, "command_buffers": 1 }, { "command_buffer_ns": 10874.99549612403, "command_buffers": 1 }, { "command_buffer_ns": 10249.961633235216, "command_buffers": 1 }, { "command_buffer_ns": 10499.963536858559, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 7.597656804136932, 7.606119652336929, 7.768880095682107, 7.796875252097379, 7.578125405416358, 8.103515938273631, 7.906901373644359, 7.617187293362804, 8.84375003806781 ], "command_buffer_throughput_us_p50": 7.768880095682107, "command_buffer_throughput_us_p90": 8.251562758232467, "command_buffer_latency_us": [ 11.6666778922081, 10.499963536858559, 10.750023648142815, 10.500021744519472, 10.45834505930543, 10.375049896538258, 10.87499549612403, 10.249961633235216, 10.499963536858559 ], "command_buffer_latency_us_p50": 10.499963536858559, "command_buffer_latency_us_p90": 11.033331975340843 }, "throughput": [ { "compute_ns": 483542.0, "compute_span_ns": 483542.0, "command_buffer_ns": 483583.30968767405, "calibration_cpu_ns": 1200250.0, "calibration_gpu_ticks": 1200250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485125.0, "compute_span_ns": 485125.0, "command_buffer_ns": 485124.99779462814, "calibration_cpu_ns": 1241750.0, "calibration_gpu_ticks": 1241750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 484125.0, "compute_span_ns": 484125.0, "command_buffer_ns": 484124.9901801348, "calibration_cpu_ns": 1173584.0, "calibration_gpu_ticks": 1173584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485166.0, "compute_span_ns": 485166.0, "command_buffer_ns": 485124.99779462814, "calibration_cpu_ns": 1219750.0, "calibration_gpu_ticks": 1219750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485625.0, "compute_span_ns": 485625.0, "command_buffer_ns": 485625.0016018748, "calibration_cpu_ns": 1260000.0, "calibration_gpu_ticks": 1260000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 484792.0, "compute_span_ns": 484792.0, "command_buffer_ns": 484833.31920579076, "calibration_cpu_ns": 1200667.0, "calibration_gpu_ticks": 1200667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485417.0, "compute_span_ns": 485417.0, "command_buffer_ns": 485416.6181758046, "calibration_cpu_ns": 1166875.0, "calibration_gpu_ticks": 1166875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 484542.0, "compute_span_ns": 484542.0, "command_buffer_ns": 484541.6988246143, "calibration_cpu_ns": 1278292.0, "calibration_gpu_ticks": 1278292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 543250.0, "compute_span_ns": 543250.0, "command_buffer_ns": 543250.0038295984, "calibration_cpu_ns": 1282792.0, "calibration_gpu_ticks": 1282792.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 10625.0, "compute_span_ns": 10625.0, "command_buffer_ns": 10624.993592500687, "calibration_cpu_ns": 431041.0, "calibration_gpu_ticks": 431041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10458.0, "compute_span_ns": 10458.0, "command_buffer_ns": 10416.668374091387, "calibration_cpu_ns": 475916.0, "calibration_gpu_ticks": 475916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10542.0, "compute_span_ns": 10542.0, "command_buffer_ns": 10541.640222072601, "calibration_cpu_ns": 390209.0, "calibration_gpu_ticks": 390209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10583.0, "compute_span_ns": 10583.0, "command_buffer_ns": 10583.316907286644, "calibration_cpu_ns": 389000.0, "calibration_gpu_ticks": 389000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12000.0, "compute_span_ns": 12000.0, "command_buffer_ns": 11999.974958598614, "calibration_cpu_ns": 677000.0, "calibration_gpu_ticks": 677000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10417.0, "compute_span_ns": 10417.0, "command_buffer_ns": 10416.668374091387, "calibration_cpu_ns": 400750.0, "calibration_gpu_ticks": 400750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10500.0, "compute_span_ns": 10500.0, "command_buffer_ns": 10499.963536858559, "calibration_cpu_ns": 406167.0, "calibration_gpu_ticks": 406167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10333.0, "compute_span_ns": 10333.0, "command_buffer_ns": 10291.638318449259, "calibration_cpu_ns": 381958.0, "calibration_gpu_ticks": 381958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10375.0, "compute_span_ns": 10375.0, "command_buffer_ns": 10374.991688877344, "calibration_cpu_ns": 592708.0, "calibration_gpu_ticks": 592708.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 7.55534375, 7.580078125, 7.564453125, 7.58071875, 7.587890625, 7.574875, 7.584640625, 7.57096875, 8.48828125 ], "compute_throughput_us_p50": 7.580078125, "compute_throughput_us_p90": 7.7679687500000005, "command_buffer_throughput_us": [ 7.555989213869907, 7.580078090541065, 7.564452971564606, 7.580078090541065, 7.587890650029294, 7.575520612590481, 7.584634658996947, 7.570964044134598, 8.488281309837475 ], "command_buffer_throughput_us_p50": 7.580078090541065, "command_buffer_throughput_us_p90": 7.76796878199093, "compute_latency_us": [ 10.625, 10.458, 10.542, 10.583, 12.0, 10.417, 10.5, 10.333, 10.375 ], "compute_latency_us_p50": 10.5, "compute_latency_us_p90": 10.9, "command_buffer_latency_us": [ 10.624993592500687, 10.416668374091387, 10.541640222072601, 10.583316907286644, 11.999974958598614, 10.416668374091387, 10.499963536858559, 10.291638318449259, 10.374991688877344 ], "command_buffer_latency_us_p50": 10.499963536858559, "command_buffer_latency_us_p90": 10.899989865720272, "counter_control_throughput_ratio": 0.9736864101904313, "counter_control_latency_ratio": 0.9920670998070825 }, "download_ms": 0.330458, "allocation_upload_ms": 0.847834, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3758430160255955, "device_compute_slowdown": 0.582408657562484, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.260833, "warmup_ms": 200.257083, "repetitions": 2428, "throughput_us": [ 12.892590197693574, 12.175521828665568, 12.523149917627677, 12.280288714991762, 12.517538714991762, 12.326588962108733, 12.351626853377265, 12.73616845140033, 12.434805601317956 ], "latency_us": [ 327.083, 223.125, 202.75, 230.708, 336.83299999999997, 451.459, 312.833, 221.584, 211.375 ], "throughput_us_p50": 12.434805601317956, "throughput_us_p90": 12.767452800658978, "latency_us_p50": 230.708, "latency_us_p90": 359.7582, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 563875.0153593719, "command_buffers": 1 }, { "command_buffer_ns": 540416.6877269745, "command_buffers": 1 }, { "command_buffer_ns": 557625.0259764493, "command_buffers": 1 }, { "command_buffer_ns": 571999.9899156392, "command_buffers": 1 }, { "command_buffer_ns": 571499.9861083925, "command_buffers": 1 }, { "command_buffer_ns": 545500.0209622085, "command_buffers": 1 }, { "command_buffer_ns": 502000.0389777124, "command_buffers": 1 }, { "command_buffer_ns": 518499.98999387026, "command_buffers": 1 }, { "command_buffer_ns": 518625.0200495124, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 10750.023648142815, "command_buffers": 1 }, { "command_buffer_ns": 15708.326827734709, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10583.316907286644, "command_buffers": 1 }, { "command_buffer_ns": 10374.991688877344, "command_buffers": 1 }, { "command_buffer_ns": 12875.01072511077, "command_buffers": 1 }, { "command_buffer_ns": 13291.661161929369, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10416.668374091387, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 8.810547114990186, 8.444010745733976, 8.71289103088202, 8.937499842431862, 8.929687282943632, 8.523437827534508, 7.843750609026756, 8.101562343654223, 8.103515938273631 ], "command_buffer_throughput_us_p50": 8.523437827534508, "command_buffer_throughput_us_p90": 8.931249794841278, "command_buffer_latency_us": [ 10.750023648142815, 15.708326827734709, 10.25001984089613, 10.583316907286644, 10.374991688877344, 12.87501072511077, 13.291661161929369, 10.25001984089613, 10.416668374091387 ], "command_buffer_latency_us_p50": 10.583316907286644, "command_buffer_latency_us_p90": 13.774994295090437 }, "throughput": [ { "compute_ns": 485250.0, "compute_span_ns": 485250.0, "command_buffer_ns": 485250.0278502703, "calibration_cpu_ns": 1190583.0, "calibration_gpu_ticks": 1190583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 555708.0, "compute_span_ns": 555708.0, "command_buffer_ns": 555708.3059102297, "calibration_cpu_ns": 1431458.0, "calibration_gpu_ticks": 1431458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 484625.0, "compute_span_ns": 484625.0, "command_buffer_ns": 484624.99398738146, "calibration_cpu_ns": 1177375.0, "calibration_gpu_ticks": 1177375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 497417.0, "compute_span_ns": 497417.0, "command_buffer_ns": 497416.65134206414, "calibration_cpu_ns": 1392833.0, "calibration_gpu_ticks": 1392833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 485083.0, "compute_span_ns": 485083.0, "command_buffer_ns": 485083.3211094141, "calibration_cpu_ns": 1164875.0, "calibration_gpu_ticks": 1164875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 487084.0, "compute_span_ns": 487084.0, "command_buffer_ns": 487125.0130236149, "calibration_cpu_ns": 1815459.0, "calibration_gpu_ticks": 1815459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 491167.0, "compute_span_ns": 491167.0, "command_buffer_ns": 491166.6619591415, "calibration_cpu_ns": 1522542.0, "calibration_gpu_ticks": 1522542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 564500.0, "compute_span_ns": 564500.0, "command_buffer_ns": 564499.9910145998, "calibration_cpu_ns": 1331333.0, "calibration_gpu_ticks": 1331333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 563292.0, "compute_span_ns": 563292.0, "command_buffer_ns": 563333.3348669112, "calibration_cpu_ns": 1509708.0, "calibration_gpu_ticks": 1509708.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 10375.0, "compute_span_ns": 10375.0, "command_buffer_ns": 10374.991688877344, "calibration_cpu_ns": 403375.0, "calibration_gpu_ticks": 403375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12375.0, "compute_span_ns": 12375.0, "command_buffer_ns": 12374.94871020317, "calibration_cpu_ns": 475625.0, "calibration_gpu_ticks": 475625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10250.0, "compute_span_ns": 10250.0, "command_buffer_ns": 10250.01984089613, "calibration_cpu_ns": 802708.0, "calibration_gpu_ticks": 802708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11167.0, "compute_span_ns": 11167.0, "command_buffer_ns": 11208.350770175457, "calibration_cpu_ns": 520041.0, "calibration_gpu_ticks": 520041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10583.0, "compute_span_ns": 10583.0, "command_buffer_ns": 10583.316907286644, "calibration_cpu_ns": 416042.0, "calibration_gpu_ticks": 416042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10541.0, "compute_span_ns": 10541.0, "command_buffer_ns": 10500.021744519472, "calibration_cpu_ns": 409333.0, "calibration_gpu_ticks": 409333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10291.0, "compute_span_ns": 10291.0, "command_buffer_ns": 10250.01984089613, "calibration_cpu_ns": 925375.0, "calibration_gpu_ticks": 925375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10750.0, "compute_span_ns": 10750.0, "command_buffer_ns": 10749.965440481901, "calibration_cpu_ns": 438708.0, "calibration_gpu_ticks": 438708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10417.0, "compute_span_ns": 10417.0, "command_buffer_ns": 10416.668374091387, "calibration_cpu_ns": 400792.0, "calibration_gpu_ticks": 400792.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 7.58203125, 8.6829375, 7.572265625, 7.772140625, 7.579421875, 7.6106875, 7.674484375, 8.8203125, 8.8014375 ], "compute_throughput_us_p50": 7.674484375, "compute_throughput_us_p90": 8.8052125, "command_buffer_throughput_us": [ 7.582031685160473, 8.682942279847339, 7.572265531052835, 7.772135177219752, 7.579426892334595, 7.6113283284939826, 7.674479093111586, 8.820312359603122, 8.802083357295487 ], "command_buffer_throughput_us_p50": 7.674479093111586, "command_buffer_throughput_us_p90": 8.805729157757014, "compute_latency_us": [ 10.375, 12.375, 10.25, 11.167, 10.583, 10.541, 10.291, 10.75, 10.417 ], "compute_latency_us_p50": 10.541, "compute_latency_us_p90": 11.4086, "command_buffer_latency_us": [ 10.374991688877344, 12.37494871020317, 10.25001984089613, 11.208350770175457, 10.583316907286644, 10.500021744519472, 10.25001984089613, 10.749965440481901, 10.416668374091387 ], "command_buffer_latency_us_p50": 10.500021744519472, "command_buffer_latency_us_p90": 11.441670358181, "counter_control_throughput_ratio": 0.8929880738856327, "counter_control_latency_ratio": 1.0 }, "download_ms": 0.323916, "allocation_upload_ms": 0.718125, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-y1v585ee/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "53243cd32bb4620fa91e934e3d500807c417fa70fc3ac1977dac07575a48be10", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.649459, "capture_ms": 0.075709, "compile_ms": 72.99925, "allocation_upload_ms": 1.588625, "cold_call_ms": 2.082875, "warmup_ms": 200.401542, "download_ms": 0.314791, "repetitions": 6026, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 5.055772154, 4.98113043478, 4.87037421175, 4.93115947561, 4.92723896449, 4.93671174909, 4.92855260538, 4.93774195154, 4.93675323598 ], "latency_us": [ 249.459, 213.083, 198.791, 197.958, 181.75, 213.416, 311.541, 217.375, 210.875 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 312583, "compute_span_ns": 312583, "calibration_cpu_ns": 743167, "calibration_gpu_ticks": 743167, "compute_passes": 1, "command_buffer_ns": 312666.699756, "command_buffers": 2 }, { "compute_ns": 315708, "compute_span_ns": 315708, "calibration_cpu_ns": 728791, "calibration_gpu_ticks": 728791, "compute_passes": 1, "command_buffer_ns": 315750.017762, "command_buffers": 2 }, { "compute_ns": 293459, "compute_span_ns": 293459, "calibration_cpu_ns": 836875, "calibration_gpu_ticks": 836875, "compute_passes": 1, "command_buffer_ns": 293541.699648, "command_buffers": 2 }, { "compute_ns": 325083, "compute_span_ns": 325083, "calibration_cpu_ns": 837208, "calibration_gpu_ticks": 837208, "compute_passes": 1, "command_buffer_ns": 325083.325151, "command_buffers": 2 }, { "compute_ns": 294209, "compute_span_ns": 294209, "calibration_cpu_ns": 686709, "calibration_gpu_ticks": 686709, "compute_passes": 1, "command_buffer_ns": 294333.323836, "command_buffers": 2 }, { "compute_ns": 321292, "compute_span_ns": 321292, "calibration_cpu_ns": 1031708, "calibration_gpu_ticks": 1031708, "compute_passes": 1, "command_buffer_ns": 321375.03149, "command_buffers": 2 }, { "compute_ns": 299375, "compute_span_ns": 299375, "calibration_cpu_ns": 954375, "calibration_gpu_ticks": 954375, "compute_passes": 1, "command_buffer_ns": 299375.038594, "command_buffers": 2 }, { "compute_ns": 308209, "compute_span_ns": 308209, "calibration_cpu_ns": 742959, "calibration_gpu_ticks": 742959, "compute_passes": 1, "command_buffer_ns": 308333.372232, "command_buffers": 2 }, { "compute_ns": 338667, "compute_span_ns": 338667, "calibration_cpu_ns": 773375, "calibration_gpu_ticks": 773375, "compute_passes": 1, "command_buffer_ns": 338666.664902, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7416, "compute_span_ns": 7416, "calibration_cpu_ns": 357708, "calibration_gpu_ticks": 357708, "compute_passes": 1, "command_buffer_ns": 7458.26400816, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 391333, "calibration_gpu_ticks": 391333, "compute_passes": 1, "command_buffer_ns": 7166.64362699, "command_buffers": 2 }, { "compute_ns": 7917, "compute_span_ns": 7917, "calibration_cpu_ns": 373167, "calibration_gpu_ticks": 373167, "compute_passes": 1, "command_buffer_ns": 8000.00270829, "command_buffers": 2 }, { "compute_ns": 8250, "compute_span_ns": 8250, "calibration_cpu_ns": 740833, "calibration_gpu_ticks": 740833, "compute_passes": 1, "command_buffer_ns": 8291.68129712, "command_buffers": 2 }, { "compute_ns": 8292, "compute_span_ns": 8292, "calibration_cpu_ns": 357875, "calibration_gpu_ticks": 357875, "compute_passes": 1, "command_buffer_ns": 8291.68129712, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 442167, "calibration_gpu_ticks": 442167, "compute_passes": 1, "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "compute_ns": 7625, "compute_span_ns": 7625, "calibration_cpu_ns": 386083, "calibration_gpu_ticks": 386083, "compute_passes": 1, "command_buffer_ns": 7708.32411945, "command_buffers": 2 }, { "compute_ns": 7709, "compute_span_ns": 7709, "calibration_cpu_ns": 368250, "calibration_gpu_ticks": 368250, "compute_passes": 1, "command_buffer_ns": 7833.35417509, "command_buffers": 2 }, { "compute_ns": 7625, "compute_span_ns": 7625, "calibration_cpu_ns": 382500, "calibration_gpu_ticks": 382500, "compute_passes": 1, "command_buffer_ns": 7666.64743423, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 328458.379954, "command_buffers": 2 }, { "command_buffer_ns": 312874.966767, "command_buffers": 2 }, { "command_buffer_ns": 320041.668601, "command_buffers": 2 }, { "command_buffer_ns": 306624.977384, "command_buffers": 2 }, { "command_buffer_ns": 305083.289277, "command_buffers": 2 }, { "command_buffer_ns": 318000.034895, "command_buffers": 2 }, { "command_buffer_ns": 331458.34459, "command_buffers": 2 }, { "command_buffer_ns": 336624.972988, "command_buffers": 2 }, { "command_buffer_ns": 327916.699462, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7249.99699742, "command_buffers": 2 }, { "command_buffer_ns": 10125.0479929, "command_buffers": 2 }, { "command_buffer_ns": 7833.29596743, "command_buffers": 2 }, { "command_buffer_ns": 7875.0308603, "command_buffers": 2 }, { "command_buffer_ns": 7499.99890104, "command_buffers": 2 }, { "command_buffer_ns": 7583.35227147, "command_buffers": 2 }, { "command_buffer_ns": 8833.30358192, "command_buffers": 2 }, { "command_buffer_ns": 11041.6440293, "command_buffers": 2 }, { "command_buffer_ns": 7791.67748988, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 5.13216218678125, 4.888671355734375, 5.0006510718906245, 4.7910152716250005, 4.766926394953125, 4.968750545234375, 5.1790366342187495, 5.2597652029375, 5.12369842909375 ], "command_buffer_throughput_us_p50": 5.0006510718906245, "command_buffer_throughput_us_p90": 5.1951823479625, "command_buffer_latency_us": [ 7.24999699742, 10.125047992899999, 7.83329596743, 7.8750308603, 7.4999989010400006, 7.58335227147, 8.83330358192, 11.0416440293, 7.7916774898800005 ], "command_buffer_latency_us_p50": 7.83329596743, "command_buffer_latency_us_p90": 10.30836720018 }, "compute_throughput_us": [ 4.884109375, 4.9329375, 4.585296875, 5.079421875, 4.597015625, 5.0201875, 4.677734375, 4.815765625, 5.291671875 ], "compute_throughput_us_p50": 4.884109375, "compute_throughput_us_p90": 5.121871875, "command_buffer_throughput_us": [ 4.8854171836875, 4.9335940275312495, 4.586589057, 5.079426955484375, 4.5989581849375, 5.02148486703125, 4.67773497803125, 4.817708941125, 5.29166663909375 ], "command_buffer_throughput_us_p50": 4.8854171836875, "command_buffer_throughput_us_p90": 5.12187489220625, "compute_latency_us": [ 7.416, 7.125, 7.917, 8.25, 8.292, 7.708, 7.625, 7.709, 7.625 ], "compute_latency_us_p50": 7.708, "compute_latency_us_p90": 8.2584, "command_buffer_latency_us": [ 7.4582640081600005, 7.16664362699, 8.00000270829, 8.29168129712, 8.29168129712, 7.708324119449999, 7.708324119449999, 7.83335417509, 7.66664743423 ], "command_buffer_latency_us_p50": 7.708324119449999, "command_buffer_latency_us_p90": 8.29168129712, "counter_control_throughput_ratio": 0.9647638339468683, "counter_control_latency_ratio": 1.0164797629737137 }, "correctness": { "max_abs_error": 2.1991937471099732e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 4.93671174909, "throughput_us_p90": 4.996058778624, "latency_us_p50": 213.083, "latency_us_p90": 261.8754 }, "slowdown": 0.3970075534246198, "device_compute_slowdown": 0.6364087978223294, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-hy2qqgbf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b8d9d984f3aaf8bfdb9269f6e0efc630605362c6ff4e852cc112881dfaf5e97b", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.105834, "capture_ms": 0.077458, "compile_ms": 64.523333, "allocation_upload_ms": 2.034625, "cold_call_ms": 9.394042, "warmup_ms": 200.249459, "download_ms": 0.897209, "repetitions": 1764, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 18.0781366213, 16.9538219955, 16.6022772109, 16.5927579365, 16.553760771, 16.7946196145, 16.8110119048, 17.7272534014, 17.4821660998 ], "latency_us": [ 282.208, 421.542, 277.5, 248.917, 282.834, 247.917, 263.416, 255.834, 216.125 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1052375, "compute_span_ns": 1052375, "calibration_cpu_ns": 1791625, "calibration_gpu_ticks": 1791625, "compute_passes": 1, "command_buffer_ns": 1052416.68643, "command_buffers": 2 }, { "compute_ns": 1038042, "compute_span_ns": 1038042, "calibration_cpu_ns": 1474541, "calibration_gpu_ticks": 1474541, "compute_passes": 1, "command_buffer_ns": 1038124.95945, "command_buffers": 2 }, { "compute_ns": 1075167, "compute_span_ns": 1075167, "calibration_cpu_ns": 1577208, "calibration_gpu_ticks": 1577208, "compute_passes": 1, "command_buffer_ns": 1075333.33357, "command_buffers": 2 }, { "compute_ns": 1047875, "compute_span_ns": 1047875, "calibration_cpu_ns": 1499209, "calibration_gpu_ticks": 1499209, "compute_passes": 1, "command_buffer_ns": 1047958.32885, "command_buffers": 2 }, { "compute_ns": 1030583, "compute_span_ns": 1030583, "calibration_cpu_ns": 1583042, "calibration_gpu_ticks": 1583042, "compute_passes": 1, "command_buffer_ns": 1030541.66539, "command_buffers": 2 }, { "compute_ns": 1088000, "compute_span_ns": 1088000, "calibration_cpu_ns": 1853833, "calibration_gpu_ticks": 1853833, "compute_passes": 1, "command_buffer_ns": 1088124.99093, "command_buffers": 2 }, { "compute_ns": 1102250, "compute_span_ns": 1102250, "calibration_cpu_ns": 1726625, "calibration_gpu_ticks": 1726625, "compute_passes": 1, "command_buffer_ns": 1102291.68786, "command_buffers": 2 }, { "compute_ns": 1045667, "compute_span_ns": 1045667, "calibration_cpu_ns": 1504792, "calibration_gpu_ticks": 1504792, "compute_passes": 1, "command_buffer_ns": 1045791.66509, "command_buffers": 2 }, { "compute_ns": 1083833, "compute_span_ns": 1083833, "calibration_cpu_ns": 1830000, "calibration_gpu_ticks": 1830000, "compute_passes": 1, "command_buffer_ns": 1083958.37015, "command_buffers": 2 } ], "latency": [ { "compute_ns": 26083, "compute_span_ns": 26083, "calibration_cpu_ns": 651375, "calibration_gpu_ticks": 651375, "compute_passes": 1, "command_buffer_ns": 26041.7000391, "command_buffers": 2 }, { "compute_ns": 20000, "compute_span_ns": 20000, "calibration_cpu_ns": 585167, "calibration_gpu_ticks": 585167, "compute_passes": 1, "command_buffer_ns": 20083.3310373, "command_buffers": 2 }, { "compute_ns": 20084, "compute_span_ns": 20084, "calibration_cpu_ns": 426209, "calibration_gpu_ticks": 426209, "compute_passes": 1, "command_buffer_ns": 20208.361093, "command_buffers": 2 }, { "compute_ns": 21084, "compute_span_ns": 21084, "calibration_cpu_ns": 614459, "calibration_gpu_ticks": 614459, "compute_passes": 1, "command_buffer_ns": 21166.6338146, "command_buffers": 2 }, { "compute_ns": 24750, "compute_span_ns": 24750, "calibration_cpu_ns": 1139042, "calibration_gpu_ticks": 1139042, "compute_passes": 1, "command_buffer_ns": 24833.3672062, "command_buffers": 2 }, { "compute_ns": 17750, "compute_span_ns": 17750, "calibration_cpu_ns": 435542, "calibration_gpu_ticks": 435542, "compute_passes": 1, "command_buffer_ns": 17833.3139047, "command_buffers": 2 }, { "compute_ns": 21875, "compute_span_ns": 21875, "calibration_cpu_ns": 613667, "calibration_gpu_ticks": 613667, "compute_passes": 1, "command_buffer_ns": 21874.9628402, "command_buffers": 2 }, { "compute_ns": 19708, "compute_span_ns": 19708, "calibration_cpu_ns": 472042, "calibration_gpu_ticks": 472042, "compute_passes": 1, "command_buffer_ns": 19791.6524485, "command_buffers": 2 }, { "compute_ns": 17542, "compute_span_ns": 17542, "calibration_cpu_ns": 430584, "calibration_gpu_ticks": 430584, "compute_passes": 1, "command_buffer_ns": 17666.6653715, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1144375.01179, "command_buffers": 2 }, { "command_buffer_ns": 1027375.05222, "command_buffers": 2 }, { "command_buffer_ns": 1105749.97962, "command_buffers": 2 }, { "command_buffer_ns": 1091791.72432, "command_buffers": 2 }, { "command_buffer_ns": 1017916.71477, "command_buffers": 2 }, { "command_buffer_ns": 1041749.95795, "command_buffers": 2 }, { "command_buffer_ns": 1034708.34438, "command_buffers": 2 }, { "command_buffer_ns": 1043958.2984, "command_buffers": 2 }, { "command_buffer_ns": 1104291.70309, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 18541.701138, "command_buffers": 2 }, { "command_buffer_ns": 26458.2922682, "command_buffers": 2 }, { "command_buffer_ns": 20250.0377782, "command_buffers": 2 }, { "command_buffer_ns": 20333.3329409, "command_buffers": 2 }, { "command_buffer_ns": 20458.3047889, "command_buffers": 2 }, { "command_buffer_ns": 20499.9814741, "command_buffers": 2 }, { "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "command_buffer_ns": 40874.9911003, "command_buffers": 2 }, { "command_buffer_ns": 19375.0020117, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 17.88085955921875, 16.0527351909375, 17.277343431562503, 17.0592456925, 15.90494866828125, 16.27734309296875, 16.1673178809375, 16.311848412499998, 17.25455786078125 ], "command_buffer_throughput_us_p50": 16.311848412499998, "command_buffer_throughput_us_p90": 17.39804665709375, "command_buffer_latency_us": [ 18.541701138, 26.458292268199997, 20.2500377782, 20.3333329409, 20.4583047889, 20.4999814741, 20.125007722499998, 40.8749911003, 19.375002011699998 ], "command_buffer_latency_us_p50": 20.3333329409, "command_buffer_latency_us_p90": 29.34163203462 }, "compute_throughput_us": [ 16.443359375, 16.21940625, 16.799484375, 16.373046875, 16.102859375, 17.0, 17.22265625, 16.338546875, 16.934890625 ], "compute_throughput_us_p50": 16.443359375, "compute_throughput_us_p90": 17.04453125, "command_buffer_throughput_us": [ 16.44401072546875, 16.22070249140625, 16.80208333703125, 16.37434888828125, 16.10221352171875, 17.001952983281253, 17.2233076228125, 16.34049476703125, 16.93684953359375 ], "command_buffer_throughput_us_p50": 16.44401072546875, "command_buffer_throughput_us_p90": 17.046223911187504, "compute_latency_us": [ 26.083, 20.0, 20.084, 21.084, 24.75, 17.75, 21.875, 19.708, 17.542 ], "compute_latency_us_p50": 20.084, "compute_latency_us_p90": 25.0166, "command_buffer_latency_us": [ 26.0417000391, 20.0833310373, 20.208361093, 21.1666338146, 24.833367206200002, 17.8333139047, 21.874962840200002, 19.791652448500002, 17.6666653715 ], "command_buffer_latency_us_p50": 20.208361093, "command_buffer_latency_us_p90": 25.075033772780003, "counter_control_throughput_ratio": 1.0017561685105716, "counter_control_latency_ratio": 0.9979418959284675 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 16.8110119048, "throughput_us_p90": 17.79743004538, "latency_us_p50": 263.416, "latency_us_p90": 310.5756 }, "torch": { "cold_call_ms": 7.455625, "warmup_ms": 200.719834, "repetitions": 628, "throughput_us": [ 45.47744108280255, 46.05812101910828, 46.99648248407644, 45.765127388535035, 45.8454076433121, 45.79889808917197, 47.35496178343949, 45.960654458598725, 47.64052547770701 ], "latency_us": [ 377.79200000000003, 347.625, 410.833, 497.16700000000003, 1015.2080000000001, 338.625, 291.25, 468.333, 285.125 ], "throughput_us_p50": 45.960654458598725, "throughput_us_p90": 47.412074522293, "latency_us_p50": 377.79200000000003, "latency_us_p90": 600.7752000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2787166.682537645, "command_buffers": 1 }, { "command_buffer_ns": 2929500.0131241977, "command_buffers": 1 }, { "command_buffer_ns": 2635375.0145062804, "command_buffers": 1 }, { "command_buffer_ns": 2846333.372872323, "command_buffers": 1 }, { "command_buffer_ns": 2644416.701514274, "command_buffers": 1 }, { "command_buffer_ns": 2802625.0074617565, "command_buffers": 1 }, { "command_buffer_ns": 2656833.326909691, "command_buffers": 1 }, { "command_buffer_ns": 3051250.008866191, "command_buffers": 1 }, { "command_buffer_ns": 2574499.9875314534, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 42791.65295884013, "command_buffers": 1 }, { "command_buffer_ns": 49916.67810827494, "command_buffers": 1 }, { "command_buffer_ns": 41958.35208520293, "command_buffers": 1 }, { "command_buffer_ns": 50249.97517466545, "command_buffers": 1 }, { "command_buffer_ns": 44666.69633984566, "command_buffers": 1 }, { "command_buffer_ns": 44749.99150261283, "command_buffers": 1 }, { "command_buffer_ns": 46500.00482797623, "command_buffers": 1 }, { "command_buffer_ns": 45041.670091450214, "command_buffers": 1 }, { "command_buffer_ns": 43041.654862463474, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 43.549479414650705, 45.77343770506559, 41.17773460166063, 44.473958951130044, 41.31901096116053, 43.791015741589945, 41.51302073296392, 47.67578138853423, 40.22656230517896 ], "command_buffer_throughput_us_p50": 43.549479414650705, "command_buffer_throughput_us_p90": 46.15390644175932, "command_buffer_latency_us": [ 42.79165295884013, 49.91667810827494, 41.95835208520293, 50.24997517466545, 44.66669633984566, 44.74999150261283, 46.50000482797623, 45.041670091450214, 43.041654862463474 ], "command_buffer_latency_us_p50": 44.74999150261283, "command_buffer_latency_us_p90": 49.98333752155304 }, "throughput": [ { "compute_ns": 2568916.0, "compute_span_ns": 2568916.0, "command_buffer_ns": 2568874.9738037586, "calibration_cpu_ns": 3429792.0, "calibration_gpu_ticks": 3429792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 3025167.0, "compute_span_ns": 3025167.0, "command_buffer_ns": 3025166.690349579, "calibration_cpu_ns": 4241375.0, "calibration_gpu_ticks": 4241375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2872459.0, "compute_span_ns": 2872459.0, "command_buffer_ns": 2872499.986551702, "calibration_cpu_ns": 3749292.0, "calibration_gpu_ticks": 3749292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2559208.0, "compute_span_ns": 2559208.0, "command_buffer_ns": 2559208.3111405373, "calibration_cpu_ns": 3517792.0, "calibration_gpu_ticks": 3517792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2812917.0, "compute_span_ns": 2812917.0, "command_buffer_ns": 2812916.6457802057, "calibration_cpu_ns": 3721083.0, "calibration_gpu_ticks": 3721083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2636125.0, "compute_span_ns": 2636125.0, "command_buffer_ns": 2636125.0202171504, "calibration_cpu_ns": 3837583.0, "calibration_gpu_ticks": 3837583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2815500.0, "compute_span_ns": 2815500.0, "command_buffer_ns": 2815500.0181868672, "calibration_cpu_ns": 3749917.0, "calibration_gpu_ticks": 3749917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2569166.0, "compute_span_ns": 2569166.0, "command_buffer_ns": 2569125.033915043, "calibration_cpu_ns": 3972209.0, "calibration_gpu_ticks": 3972209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2836875.0, "compute_span_ns": 2836875.0, "command_buffer_ns": 2836874.97721985, "calibration_cpu_ns": 3958542.0, "calibration_gpu_ticks": 3958542.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 49334.0, "compute_span_ns": 49334.0, "command_buffer_ns": 49374.99761581421, "calibration_cpu_ns": 825292.0, "calibration_gpu_ticks": 825292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44167.0, "compute_span_ns": 44167.0, "command_buffer_ns": 44208.3110101521, "calibration_cpu_ns": 573417.0, "calibration_gpu_ticks": 573417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43459.0, "compute_span_ns": 43459.0, "command_buffer_ns": 43500.04019215703, "calibration_cpu_ns": 520000.0, "calibration_gpu_ticks": 520000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44583.0, "compute_span_ns": 44583.0, "command_buffer_ns": 44541.66628420353, "calibration_cpu_ns": 449917.0, "calibration_gpu_ticks": 449917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45166.0, "compute_span_ns": 45166.0, "command_buffer_ns": 45125.0234618783, "calibration_cpu_ns": 1347333.0, "calibration_gpu_ticks": 1347333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44083.0, "compute_span_ns": 44083.0, "command_buffer_ns": 44083.33916217089, "calibration_cpu_ns": 478583.0, "calibration_gpu_ticks": 478583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43750.0, "compute_span_ns": 43750.0, "command_buffer_ns": 43750.04209578037, "calibration_cpu_ns": 463375.0, "calibration_gpu_ticks": 463375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44584.0, "compute_span_ns": 44584.0, "command_buffer_ns": 44625.019654631615, "calibration_cpu_ns": 944792.0, "calibration_gpu_ticks": 944792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43458.0, "compute_span_ns": 43458.0, "command_buffer_ns": 43416.686821728945, "calibration_cpu_ns": 522208.0, "calibration_gpu_ticks": 522208.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 40.1393125, 47.268234375, 44.882171875, 39.987625, 43.951828125, 41.189453125, 43.9921875, 40.14321875, 44.326171875 ], "compute_throughput_us_p50": 43.951828125, "compute_throughput_us_p90": 45.359384375, "command_buffer_throughput_us": [ 40.13867146568373, 47.26822953671217, 44.882812289870344, 39.987629861570895, 43.951822590315714, 41.189453440892976, 43.9921877841698, 40.142578654922545, 44.32617151906015 ], "command_buffer_throughput_us_p50": 43.951822590315714, "command_buffer_throughput_us_p90": 45.35989573923871, "compute_latency_us": [ 49.334, 44.167, 43.459, 44.583, 45.166, 44.083, 43.75, 44.584, 43.458 ], "compute_latency_us_p50": 44.167, "compute_latency_us_p90": 45.9996, "command_buffer_latency_us": [ 49.37499761581421, 44.2083110101521, 43.50004019215703, 44.54166628420353, 45.1250234618783, 44.08333916217089, 43.75004209578037, 44.625019654631615, 43.416686821728945 ], "command_buffer_latency_us_p50": 44.2083110101521, "command_buffer_latency_us_p90": 45.97501829266548, "counter_control_throughput_ratio": 1.032656315684176, "counter_control_latency_ratio": 0.990749667230974 }, "download_ms": 0.36875, "allocation_upload_ms": 1.827208, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3657696371565668, "device_compute_slowdown": 0.3741223079102583, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.298667, "warmup_ms": 201.122125, "repetitions": 589, "throughput_us": [ 46.27801358234296, 46.16843463497453, 46.534592529711375, 47.36622750424448, 45.89643463497453, 46.57130730050934, 46.24455348047538, 45.935483870967744, 45.84939049235994 ], "latency_us": [ 412.292, 263.291, 273.0, 260.91700000000003, 273.083, 282.375, 288.95799999999997, 327.29200000000003, 304.666 ], "throughput_us_p50": 46.24455348047538, "throughput_us_p90": 46.73029134125637, "latency_us_p50": 282.375, "latency_us_p90": 344.29200000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2659999.9982863665, "command_buffers": 1 }, { "command_buffer_ns": 2581249.9807216227, "command_buffers": 1 }, { "command_buffer_ns": 2617000.020109117, "command_buffers": 1 }, { "command_buffer_ns": 2591541.677247733, "command_buffers": 1 }, { "command_buffer_ns": 2605749.9926537275, "command_buffers": 1 }, { "command_buffer_ns": 2587124.996352941, "command_buffers": 1 }, { "command_buffer_ns": 2599874.9770224094, "command_buffers": 1 }, { "command_buffer_ns": 2626791.7128279805, "command_buffers": 1 }, { "command_buffer_ns": 2596708.363853395, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 43999.9857917428, "command_buffers": 1 }, { "command_buffer_ns": 44249.987695366144, "command_buffers": 1 }, { "command_buffer_ns": 44249.987695366144, "command_buffers": 1 }, { "command_buffer_ns": 46583.35819840431, "command_buffers": 1 }, { "command_buffer_ns": 49541.646149009466, "command_buffers": 1 }, { "command_buffer_ns": 43791.6605733335, "command_buffers": 1 }, { "command_buffer_ns": 43625.012040138245, "command_buffers": 1 }, { "command_buffer_ns": 44333.34106579423, "command_buffers": 1 }, { "command_buffer_ns": 45041.670091450214, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 41.562499973224476, 40.332030948775355, 40.890625314204954, 40.492838706995826, 40.71484363521449, 40.4238280680147, 40.62304651597515, 41.043620512937196, 40.5735681852093 ], "command_buffer_throughput_us_p50": 40.62304651597515, "command_buffer_throughput_us_p90": 41.14739640499465, "command_buffer_latency_us": [ 43.9999857917428, 44.249987695366144, 44.249987695366144, 46.58335819840431, 49.541646149009466, 43.7916605733335, 43.625012040138245, 44.33334106579423, 45.041670091450214 ], "command_buffer_latency_us_p50": 44.249987695366144, "command_buffer_latency_us_p90": 47.17501578852534 }, "throughput": [ { "compute_ns": 2653250.0, "compute_span_ns": 2653250.0, "command_buffer_ns": 2653250.005096197, "calibration_cpu_ns": 3696042.0, "calibration_gpu_ticks": 3696042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2593667.0, "compute_span_ns": 2593667.0, "command_buffer_ns": 2593666.664324701, "calibration_cpu_ns": 3523000.0, "calibration_gpu_ticks": 3523000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2613000.0, "compute_span_ns": 2613000.0, "command_buffer_ns": 2612999.9896511436, "calibration_cpu_ns": 3680625.0, "calibration_gpu_ticks": 3680625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2597125.0, "compute_span_ns": 2597125.0, "command_buffer_ns": 2597125.0142902136, "calibration_cpu_ns": 3552084.0, "calibration_gpu_ticks": 3552084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2653750.0, "compute_span_ns": 2653750.0, "command_buffer_ns": 2653749.950695783, "calibration_cpu_ns": 3631750.0, "calibration_gpu_ticks": 3631750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2613584.0, "compute_span_ns": 2613584.0, "command_buffer_ns": 2613625.0235140324, "calibration_cpu_ns": 3491417.0, "calibration_gpu_ticks": 3491417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2602000.0, "compute_span_ns": 2602000.0, "command_buffer_ns": 2601999.9640993774, "calibration_cpu_ns": 3584166.0, "calibration_gpu_ticks": 3584166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2604166.0, "compute_span_ns": 2604166.0, "command_buffer_ns": 2604125.0093840063, "calibration_cpu_ns": 3622417.0, "calibration_gpu_ticks": 3622417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2566542.0, "compute_span_ns": 2566542.0, "command_buffer_ns": 2566541.6615083814, "calibration_cpu_ns": 3506416.0, "calibration_gpu_ticks": 3506416.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 44042.0, "compute_span_ns": 44042.0, "command_buffer_ns": 44083.33916217089, "calibration_cpu_ns": 602500.0, "calibration_gpu_ticks": 602500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44958.0, "compute_span_ns": 44958.0, "command_buffer_ns": 44916.64003580809, "calibration_cpu_ns": 528750.0, "calibration_gpu_ticks": 528750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44750.0, "compute_span_ns": 44750.0, "command_buffer_ns": 44749.99150261283, "calibration_cpu_ns": 455375.0, "calibration_gpu_ticks": 455375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44958.0, "compute_span_ns": 44958.0, "command_buffer_ns": 44958.37492868304, "calibration_cpu_ns": 455291.0, "calibration_gpu_ticks": 455291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44042.0, "compute_span_ns": 44042.0, "command_buffer_ns": 44041.662476956844, "calibration_cpu_ns": 488000.0, "calibration_gpu_ticks": 488000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44041.0, "compute_span_ns": 44041.0, "command_buffer_ns": 43999.9857917428, "calibration_cpu_ns": 445375.0, "calibration_gpu_ticks": 445375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47709.0, "compute_span_ns": 47709.0, "command_buffer_ns": 47750.01434609294, "calibration_cpu_ns": 757250.0, "calibration_gpu_ticks": 757250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44916.0, "compute_span_ns": 44916.0, "command_buffer_ns": 44875.02155825496, "calibration_cpu_ns": 724292.0, "calibration_gpu_ticks": 724292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44792.0, "compute_span_ns": 44792.0, "command_buffer_ns": 44833.344873040915, "calibration_cpu_ns": 481917.0, "calibration_gpu_ticks": 481917.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 41.45703125, 40.526046875, 40.828125, 40.580078125, 41.46484375, 40.83725, 40.65625, 40.69009375, 40.10221875 ], "compute_throughput_us_p50": 40.69009375, "compute_throughput_us_p90": 41.45859375, "command_buffer_throughput_us": [ 41.45703132962808, 40.52604163007345, 40.82812483829912, 40.58007834828459, 41.46484297962161, 40.837890992406756, 40.65624943905277, 40.6894532716251, 40.10221346106846 ], "command_buffer_throughput_us_p50": 40.6894532716251, "command_buffer_throughput_us_p90": 41.458593659626786, "compute_latency_us": [ 44.042, 44.958, 44.75, 44.958, 44.042, 44.041, 47.709, 44.916, 44.792 ], "compute_latency_us_p50": 44.792, "compute_latency_us_p90": 45.5082, "command_buffer_latency_us": [ 44.08333916217089, 44.916640035808086, 44.74999150261283, 44.95837492868304, 44.041662476956844, 43.9999857917428, 47.75001434609294, 44.87502155825496, 44.833344873040915 ], "command_buffer_latency_us_p50": 44.833344873040915, "command_buffer_latency_us_p90": 45.51670281216502, "counter_control_throughput_ratio": 1.0008173420244237, "counter_control_latency_ratio": 1.004757189284029 }, "download_ms": 0.426125, "allocation_upload_ms": 1.010417, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-bwz4kdxy/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "83a70ba0e4d1dab6671da56395e0b13750f8d9c9277e4782c4e371450de131c7", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.900041, "capture_ms": 0.073584, "compile_ms": 63.341333, "allocation_upload_ms": 2.824459, "cold_call_ms": 4.789875, "warmup_ms": 200.381583, "download_ms": 1.467, "repetitions": 1758, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 17.084447099, 17.1710989761, 17.081129124, 20.9672690557, 18.7458993174, 18.0834755404, 20.8624385666, 17.5531382253, 17.4598737201 ], "latency_us": [ 316, 214.875, 210.041, 228.208, 245.834, 248.25, 279.708, 238.667, 313.792 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1034083, "compute_span_ns": 1034083, "calibration_cpu_ns": 1613209, "calibration_gpu_ticks": 1613209, "compute_passes": 1, "command_buffer_ns": 1034166.72209, "command_buffers": 2 }, { "compute_ns": 1046500, "compute_span_ns": 1046500, "calibration_cpu_ns": 1498041, "calibration_gpu_ticks": 1498041, "compute_passes": 1, "command_buffer_ns": 1046583.34749, "command_buffers": 2 }, { "compute_ns": 1043583, "compute_span_ns": 1043583, "calibration_cpu_ns": 1603750, "calibration_gpu_ticks": 1603750, "compute_passes": 1, "command_buffer_ns": 1043583.32464, "command_buffers": 2 }, { "compute_ns": 1122334, "compute_span_ns": 1122334, "calibration_cpu_ns": 1885625, "calibration_gpu_ticks": 1885625, "compute_passes": 1, "command_buffer_ns": 1122375.01889, "command_buffers": 2 }, { "compute_ns": 1135875, "compute_span_ns": 1135875, "calibration_cpu_ns": 2138125, "calibration_gpu_ticks": 2138125, "compute_passes": 1, "command_buffer_ns": 1135916.68196, "command_buffers": 2 }, { "compute_ns": 1067583, "compute_span_ns": 1067583, "calibration_cpu_ns": 1514667, "calibration_gpu_ticks": 1514667, "compute_passes": 1, "command_buffer_ns": 1067541.65608, "command_buffers": 2 }, { "compute_ns": 1057875, "compute_span_ns": 1057875, "calibration_cpu_ns": 1537834, "calibration_gpu_ticks": 1537834, "compute_passes": 1, "command_buffer_ns": 1057874.99342, "command_buffers": 2 }, { "compute_ns": 1114709, "compute_span_ns": 1114709, "calibration_cpu_ns": 1904375, "calibration_gpu_ticks": 1904375, "compute_passes": 1, "command_buffer_ns": 1114833.34331, "command_buffers": 2 }, { "compute_ns": 1080042, "compute_span_ns": 1080042, "calibration_cpu_ns": 1796208, "calibration_gpu_ticks": 1796208, "compute_passes": 1, "command_buffer_ns": 1080124.98822, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18750, "compute_span_ns": 18750, "calibration_cpu_ns": 442833, "calibration_gpu_ticks": 442833, "compute_passes": 1, "command_buffer_ns": 18750.0263564, "command_buffers": 2 }, { "compute_ns": 25541, "compute_span_ns": 25541, "calibration_cpu_ns": 778542, "calibration_gpu_ticks": 778542, "compute_passes": 1, "command_buffer_ns": 25583.372917, "command_buffers": 2 }, { "compute_ns": 20084, "compute_span_ns": 20084, "calibration_cpu_ns": 427875, "calibration_gpu_ticks": 427875, "compute_passes": 1, "command_buffer_ns": 20208.361093, "command_buffers": 2 }, { "compute_ns": 19917, "compute_span_ns": 19917, "calibration_cpu_ns": 547125, "calibration_gpu_ticks": 547125, "compute_passes": 1, "command_buffer_ns": 20041.6543521, "command_buffers": 2 }, { "compute_ns": 20042, "compute_span_ns": 20042, "calibration_cpu_ns": 457458, "calibration_gpu_ticks": 457458, "compute_passes": 1, "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "compute_ns": 19792, "compute_span_ns": 19792, "calibration_cpu_ns": 411625, "calibration_gpu_ticks": 411625, "compute_passes": 1, "command_buffer_ns": 19916.6825041, "command_buffers": 2 }, { "compute_ns": 26542, "compute_span_ns": 26542, "calibration_cpu_ns": 658542, "calibration_gpu_ticks": 658542, "compute_passes": 1, "command_buffer_ns": 26541.6456386, "command_buffers": 2 }, { "compute_ns": 20458, "compute_span_ns": 20458, "calibration_cpu_ns": 437625, "calibration_gpu_ticks": 437625, "compute_passes": 1, "command_buffer_ns": 20499.9814741, "command_buffers": 2 }, { "compute_ns": 28833, "compute_span_ns": 28833, "calibration_cpu_ns": 2102167, "calibration_gpu_ticks": 2102167, "compute_passes": 1, "command_buffer_ns": 28916.6346192, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1148083.36366, "command_buffers": 2 }, { "command_buffer_ns": 1125958.34071, "command_buffers": 2 }, { "command_buffer_ns": 1033416.65817, "command_buffers": 2 }, { "command_buffer_ns": 1046791.67271, "command_buffers": 2 }, { "command_buffer_ns": 1059791.65528, "command_buffers": 2 }, { "command_buffer_ns": 1139124.97181, "command_buffers": 2 }, { "command_buffer_ns": 1064583.36813, "command_buffers": 2 }, { "command_buffer_ns": 1089124.99854, "command_buffers": 2 }, { "command_buffer_ns": 1069791.67322, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 17958.3439603, "command_buffers": 2 }, { "command_buffer_ns": 22374.9666475, "command_buffers": 2 }, { "command_buffer_ns": 20125.0077225, "command_buffers": 2 }, { "command_buffer_ns": 27875.0085272, "command_buffers": 2 }, { "command_buffer_ns": 20916.6901186, "command_buffers": 2 }, { "command_buffer_ns": 23541.6810028, "command_buffers": 2 }, { "command_buffer_ns": 27374.9465123, "command_buffers": 2 }, { "command_buffer_ns": 19916.6825041, "command_buffers": 2 }, { "command_buffer_ns": 20083.3310373, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 17.9388025571875, 17.59309907359375, 16.14713528390625, 16.35611988609375, 16.55924461375, 17.79882768453125, 16.63411512703125, 17.0175781021875, 16.7154948940625 ], "command_buffer_throughput_us_p50": 16.7154948940625, "command_buffer_throughput_us_p90": 17.8268226590625, "command_buffer_latency_us": [ 17.958343960300002, 22.374966647500003, 20.125007722499998, 27.8750085272, 20.9166901186, 23.5416810028, 27.3749465123, 19.916682504100002, 20.0833310373 ], "command_buffer_latency_us_p50": 20.9166901186, "command_buffer_latency_us_p90": 27.47495891528 }, "compute_throughput_us": [ 16.157546875, 16.3515625, 16.305984375, 17.53646875, 17.748046875, 16.680984375, 16.529296875, 17.417328125, 16.87565625 ], "compute_throughput_us_p50": 16.680984375, "compute_throughput_us_p90": 17.578784375, "command_buffer_throughput_us": [ 16.158855032656252, 16.35286480453125, 16.3059894475, 17.537109670156248, 17.748698155625, 16.680338376250003, 16.5292967721875, 17.41927098921875, 16.8769529409375 ], "command_buffer_throughput_us_p50": 16.680338376250003, "command_buffer_throughput_us_p90": 17.57942736725, "compute_latency_us": [ 18.75, 25.541, 20.084, 19.917, 20.042, 19.792, 26.542, 20.458, 28.833 ], "compute_latency_us_p50": 20.084, "compute_latency_us_p90": 27.000200000000003, "command_buffer_latency_us": [ 18.750026356400003, 25.583372917000002, 20.208361093, 20.0416543521, 20.125007722499998, 19.916682504100002, 26.5416456386, 20.4999814741, 28.9166346192 ], "command_buffer_latency_us_p50": 20.208361093, "command_buffer_latency_us_p90": 27.01664343472, "counter_control_throughput_ratio": 1.0096591843614724, "counter_control_latency_ratio": 1.0041417807957813 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 17.5531382253, "throughput_us_p90": 20.883404664420002, "latency_us_p50": 245.834, "latency_us_p90": 314.23359999999997 }, "slowdown": 0.3795720123605691, "device_compute_slowdown": 0.40995197694770613, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-7cd1zkdh/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "4f0c4e70dcb06b507aa82e242c8819acd3628d5f79a065eb6c2f6b73ec6f396c", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 22.831958, "capture_ms": 0.079083, "compile_ms": 63.027209, "allocation_upload_ms": 7.086292, "cold_call_ms": 5.285792, "warmup_ms": 200.539875, "download_ms": 3.696125, "repetitions": 382, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 77.163065445, 77.0316308901, 76.5173429319, 77.457460733, 76.7750863874, 77.0513743455, 77.0297774869, 77.1367801047, 76.7993010471 ], "latency_us": [ 1080.333, 326.792, 299.666, 338.625, 338.666, 312.625, 360.667, 309, 333.958 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4781416, "compute_span_ns": 4781416, "calibration_cpu_ns": 5484625, "calibration_gpu_ticks": 5484625, "compute_passes": 1, "command_buffer_ns": 4781374.99886, "command_buffers": 2 }, { "compute_ns": 4754958, "compute_span_ns": 4754958, "calibration_cpu_ns": 5397458, "calibration_gpu_ticks": 5397458, "compute_passes": 1, "command_buffer_ns": 4754916.70659, "command_buffers": 2 }, { "compute_ns": 4784250, "compute_span_ns": 4784250, "calibration_cpu_ns": 5471667, "calibration_gpu_ticks": 5471667, "compute_passes": 1, "command_buffer_ns": 4784375.0217, "command_buffers": 2 }, { "compute_ns": 4773166, "compute_span_ns": 4773166, "calibration_cpu_ns": 5435166, "calibration_gpu_ticks": 5435166, "compute_passes": 1, "command_buffer_ns": 4773208.34761, "command_buffers": 2 }, { "compute_ns": 4891167, "compute_span_ns": 4891167, "calibration_cpu_ns": 5687250, "calibration_gpu_ticks": 5687250, "compute_passes": 1, "command_buffer_ns": 4891249.99149, "command_buffers": 2 }, { "compute_ns": 4873958, "compute_span_ns": 4873958, "calibration_cpu_ns": 5609750, "calibration_gpu_ticks": 5609750, "compute_passes": 1, "command_buffer_ns": 4873958.29987, "command_buffers": 2 }, { "compute_ns": 4854750, "compute_span_ns": 4854750, "calibration_cpu_ns": 5483875, "calibration_gpu_ticks": 5483875, "compute_passes": 1, "command_buffer_ns": 4854750.0046, "command_buffers": 2 }, { "compute_ns": 4907500, "compute_span_ns": 4907500, "calibration_cpu_ns": 5632958, "calibration_gpu_ticks": 5632958, "compute_passes": 1, "command_buffer_ns": 4907541.67549, "command_buffers": 2 }, { "compute_ns": 4831917, "compute_span_ns": 4831917, "calibration_cpu_ns": 5561084, "calibration_gpu_ticks": 5561084, "compute_passes": 1, "command_buffer_ns": 4831958.32931, "command_buffers": 2 } ], "latency": [ { "compute_ns": 111833, "compute_span_ns": 111833, "calibration_cpu_ns": 1004958, "calibration_gpu_ticks": 1004958, "compute_passes": 1, "command_buffer_ns": 111791.65449, "command_buffers": 2 }, { "compute_ns": 75459, "compute_span_ns": 75459, "calibration_cpu_ns": 483125, "calibration_gpu_ticks": 483125, "compute_passes": 1, "command_buffer_ns": 75499.9928176, "command_buffers": 2 }, { "compute_ns": 117583, "compute_span_ns": 117583, "calibration_cpu_ns": 1029000, "calibration_gpu_ticks": 1029000, "compute_passes": 1, "command_buffer_ns": 117541.640066, "command_buffers": 2 }, { "compute_ns": 99625, "compute_span_ns": 99625, "calibration_cpu_ns": 651917, "calibration_gpu_ticks": 651917, "compute_passes": 1, "command_buffer_ns": 99750.0028461, "command_buffers": 2 }, { "compute_ns": 76500, "compute_span_ns": 76500, "calibration_cpu_ns": 497375, "calibration_gpu_ticks": 497375, "compute_passes": 1, "command_buffer_ns": 76541.6189097, "command_buffers": 2 }, { "compute_ns": 92458, "compute_span_ns": 92458, "calibration_cpu_ns": 568042, "calibration_gpu_ticks": 568042, "compute_passes": 1, "command_buffer_ns": 92541.6825339, "command_buffers": 2 }, { "compute_ns": 103958, "compute_span_ns": 103958, "calibration_cpu_ns": 831333, "calibration_gpu_ticks": 831333, "compute_passes": 1, "command_buffer_ns": 103958.358523, "command_buffers": 2 }, { "compute_ns": 76875, "compute_span_ns": 76875, "calibration_cpu_ns": 493542, "calibration_gpu_ticks": 493542, "compute_passes": 1, "command_buffer_ns": 76875.0323914, "command_buffers": 2 }, { "compute_ns": 113125, "compute_span_ns": 113125, "calibration_cpu_ns": 706334, "calibration_gpu_ticks": 706334, "compute_passes": 1, "command_buffer_ns": 113166.694064, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4841624.99197, "command_buffers": 2 }, { "command_buffer_ns": 4808583.29684, "command_buffers": 2 }, { "command_buffer_ns": 4830208.37419, "command_buffers": 2 }, { "command_buffer_ns": 4745624.95947, "command_buffers": 2 }, { "command_buffer_ns": 4974583.33848, "command_buffers": 2 }, { "command_buffer_ns": 4906541.72609, "command_buffers": 2 }, { "command_buffer_ns": 4869750.06061, "command_buffers": 2 }, { "command_buffer_ns": 4826291.63889, "command_buffers": 2 }, { "command_buffer_ns": 4808125.02792, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 121666.642372, "command_buffers": 2 }, { "command_buffer_ns": 75458.3743401, "command_buffers": 2 }, { "command_buffer_ns": 93000.009656, "command_buffers": 2 }, { "command_buffer_ns": 100958.393887, "command_buffers": 2 }, { "command_buffer_ns": 76124.9684729, "command_buffers": 2 }, { "command_buffer_ns": 117874.937132, "command_buffers": 2 }, { "command_buffer_ns": 79833.3785497, "command_buffers": 2 }, { "command_buffer_ns": 75625.0228733, "command_buffers": 2 }, { "command_buffer_ns": 94124.9891184, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 75.65039049953124, 75.134114013125, 75.47200584671874, 74.15038999171875, 77.72786466375001, 76.66471447015626, 76.08984469703125, 75.41080685765625, 75.12695356125 ], "command_buffer_throughput_us_p50": 75.47200584671874, "command_buffer_throughput_us_p90": 76.87734450887501, "command_buffer_latency_us": [ 121.666642372, 75.4583743401, 93.00000965599999, 100.958393887, 76.1249684729, 117.874937132, 79.8333785497, 75.6250228733, 94.1249891184 ], "command_buffer_latency_us_p50": 93.00000965599999, "command_buffer_latency_us_p90": 118.63327818 }, "compute_throughput_us": [ 74.709625, 74.29621875, 74.75390625, 74.58071875, 76.424484375, 76.15559375, 75.85546875, 76.6796875, 75.498703125 ], "compute_throughput_us_p50": 75.498703125, "compute_throughput_us_p90": 76.475525, "command_buffer_throughput_us": [ 74.70898435718749, 74.29557354046874, 74.75585971406251, 74.58138043140625, 76.42578111703125, 76.15559843546875, 75.855468821875, 76.68033867953126, 75.49934889546874 ], "command_buffer_throughput_us_p50": 75.49934889546874, "command_buffer_throughput_us_p90": 76.47669262953126, "compute_latency_us": [ 111.833, 75.459, 117.583, 99.625, 76.5, 92.458, 103.958, 76.875, 113.125 ], "compute_latency_us_p50": 99.625, "compute_latency_us_p90": 114.0166, "command_buffer_latency_us": [ 111.79165449, 75.4999928176, 117.54164006600001, 99.7500028461, 76.5416189097, 92.5416825339, 103.958358523, 76.8750323914, 113.166694064 ], "command_buffer_latency_us_p50": 99.7500028461, "command_buffer_latency_us_p90": 114.0416832644, "counter_control_throughput_ratio": 0.9933591869713976, "counter_control_latency_ratio": 1.0054732428158355 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 77.0316308901, "throughput_us_p90": 77.2219445026, "latency_us_p50": 333.958, "latency_us_p90": 504.60020000000014 }, "torch": { "cold_call_ms": 8.562375, "warmup_ms": 202.171417, "repetitions": 106, "throughput_us": [ 282.64740566037733, 279.89700943396224, 281.07390566037736, 282.1057358490566, 279.44654716981137, 282.1619433962264, 277.89111320754716, 278.04245283018867, 278.77240566037733 ], "latency_us": [ 1229.917, 525.208, 551.2909999999999, 812.2080000000001, 505.12500000000006, 501.25, 508.87499999999994, 517.667, 536.1669999999999 ], "throughput_us_p50": 279.89700943396224, "throughput_us_p90": 282.25903584905654, "latency_us_p50": 525.208, "latency_us_p90": 895.7498000000002, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17369874.9858886, "command_buffers": 1 }, { "command_buffer_ns": 17438083.36323127, "command_buffers": 1 }, { "command_buffer_ns": 17252749.99625981, "command_buffers": 1 }, { "command_buffer_ns": 17332000.0176318, "command_buffers": 1 }, { "command_buffer_ns": 17193250.00885874, "command_buffers": 1 }, { "command_buffer_ns": 17304749.984759837, "command_buffers": 1 }, { "command_buffer_ns": 17340250.02224371, "command_buffers": 1 }, { "command_buffer_ns": 17311166.680883616, "command_buffers": 1 }, { "command_buffer_ns": 17311375.006102026, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 273166.6900217533, "command_buffers": 1 }, { "command_buffer_ns": 283333.35649222136, "command_buffers": 1 }, { "command_buffer_ns": 279875.0065267086, "command_buffers": 1 }, { "command_buffer_ns": 283125.03127381206, "command_buffers": 1 }, { "command_buffer_ns": 283250.00312179327, "command_buffers": 1 }, { "command_buffer_ns": 282749.9993145466, "command_buffers": 1 }, { "command_buffer_ns": 281458.31311121583, "command_buffers": 1 }, { "command_buffer_ns": 271124.9981075525, "command_buffers": 1 }, { "command_buffer_ns": 287208.3568945527, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 271.4042966545094, 272.4700525504886, 269.5742186915595, 270.81250027549686, 268.6445313884178, 270.38671851187246, 270.94140659755794, 270.4869793888065, 270.49023447034415 ], "command_buffer_throughput_us_p50": 270.49023447034415, "command_buffer_throughput_us_p90": 271.6174478337052, "command_buffer_latency_us": [ 273.1666900217533, 283.33335649222136, 279.8750065267086, 283.12503127381206, 283.25000312179327, 282.7499993145466, 281.45831311121583, 271.1249981075525, 287.2083568945527 ], "command_buffer_latency_us_p50": 282.7499993145466, "command_buffer_latency_us_p90": 284.1083565726876 }, "throughput": [ { "compute_ns": 17251708.0, "compute_span_ns": 17251708.0, "command_buffer_ns": 17251666.635274887, "calibration_cpu_ns": 18290250.0, "calibration_gpu_ticks": 18290250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17283917.0, "compute_span_ns": 17283917.0, "command_buffer_ns": 17283958.32469687, "calibration_cpu_ns": 18291583.0, "calibration_gpu_ticks": 18291583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17414917.0, "compute_span_ns": 17414917.0, "command_buffer_ns": 17414958.332665265, "calibration_cpu_ns": 18414917.0, "calibration_gpu_ticks": 18414917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17271334.0, "compute_span_ns": 17271334.0, "command_buffer_ns": 17271374.992560595, "calibration_cpu_ns": 18560667.0, "calibration_gpu_ticks": 18560667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17618583.0, "compute_span_ns": 17618583.0, "command_buffer_ns": 17618541.66397825, "calibration_cpu_ns": 19067958.0, "calibration_gpu_ticks": 19067958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17635875.0, "compute_span_ns": 17635875.0, "command_buffer_ns": 17635875.032283366, "calibration_cpu_ns": 18876292.0, "calibration_gpu_ticks": 18876292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17440833.0, "compute_span_ns": 17440833.0, "command_buffer_ns": 17440833.325963467, "calibration_cpu_ns": 18493708.0, "calibration_gpu_ticks": 18493708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17347083.0, "compute_span_ns": 17347083.0, "command_buffer_ns": 17347041.69211909, "calibration_cpu_ns": 18683167.0, "calibration_gpu_ticks": 18683167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17339875.0, "compute_span_ns": 17339875.0, "command_buffer_ns": 17339874.990284443, "calibration_cpu_ns": 18588750.0, "calibration_gpu_ticks": 18588750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 272958.0, "compute_span_ns": 272958.0, "command_buffer_ns": 272916.68811812997, "calibration_cpu_ns": 713250.0, "calibration_gpu_ticks": 713250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271875.0, "compute_span_ns": 271875.0, "command_buffer_ns": 271875.00381842256, "calibration_cpu_ns": 715750.0, "calibration_gpu_ticks": 715750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 272625.0, "compute_span_ns": 272625.0, "command_buffer_ns": 272625.0095292926, "calibration_cpu_ns": 774750.0, "calibration_gpu_ticks": 774750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271250.0, "compute_span_ns": 271250.0, "command_buffer_ns": 271249.96995553374, "calibration_cpu_ns": 775041.0, "calibration_gpu_ticks": 775041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275416.0, "compute_span_ns": 275416.0, "command_buffer_ns": 275374.97226148844, "calibration_cpu_ns": 840583.0, "calibration_gpu_ticks": 840583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 272584.0, "compute_span_ns": 272584.0, "command_buffer_ns": 272625.0095292926, "calibration_cpu_ns": 914209.0, "calibration_gpu_ticks": 914209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271750.0, "compute_span_ns": 271750.0, "command_buffer_ns": 271750.03197044134, "calibration_cpu_ns": 726292.0, "calibration_gpu_ticks": 726292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 273541.0, "compute_span_ns": 273541.0, "command_buffer_ns": 273499.9870881438, "calibration_cpu_ns": 679000.0, "calibration_gpu_ticks": 679000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 309958.0, "compute_span_ns": 309958.0, "command_buffer_ns": 309958.35550129414, "calibration_cpu_ns": 911000.0, "calibration_gpu_ticks": 911000.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 269.5579375, 270.061203125, 272.108078125, 269.86459375, 275.290359375, 275.560546875, 272.513015625, 271.048171875, 270.935546875 ], "compute_throughput_us_p50": 271.048171875, "compute_throughput_us_p90": 275.344396875, "command_buffer_throughput_us": [ 269.5572911761701, 270.0618488233886, 272.10872394789476, 269.8652342587593, 275.28971349966014, 275.5605473794276, 272.5130207181792, 271.0475264393608, 270.9355467231944 ], "command_buffer_throughput_us_p50": 271.0475264393608, "command_buffer_throughput_us_p90": 275.3438802756136, "compute_latency_us": [ 272.958, 271.875, 272.625, 271.25, 275.416, 272.584, 271.75, 273.541, 309.958 ], "compute_latency_us_p50": 272.625, "compute_latency_us_p90": 282.3244, "command_buffer_latency_us": [ 272.91668811812997, 271.87500381842256, 272.6250095292926, 271.24996995553374, 275.37497226148844, 272.6250095292926, 271.75003197044134, 273.4999870881438, 309.95835550129414 ], "command_buffer_latency_us_p50": 272.6250095292926, "command_buffer_latency_us_p90": 282.2916489094496, "counter_control_throughput_ratio": 1.0020723624176695, "counter_control_latency_ratio": 0.9721975965630663 }, "download_ms": 0.824542, "allocation_upload_ms": 18.441458, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.2752141977003671, "device_compute_slowdown": 0.2785434876860854, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.580542, "warmup_ms": 201.323, "repetitions": 107, "throughput_us": [ 292.3107476635514, 343.63862616822433, 313.7102803738318, 308.5093457943925, 314.5525700934579, 301.86292523364483, 304.62733644859816, 305.05412149532714, 313.8446261682243 ], "latency_us": [ 1103.9170000000001, 1464.875, 544.0409999999999, 579.75, 587.584, 1610.6670000000001, 622.5, 648.916, 541.5 ], "throughput_us_p50": 308.5093457943925, "throughput_us_p90": 320.3697813084112, "latency_us_p50": 622.5, "latency_us_p90": 1494.0334, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 18876833.375543356, "command_buffers": 1 }, { "command_buffer_ns": 19422125.013079494, "command_buffers": 1 }, { "command_buffer_ns": 19006874.99437481, "command_buffers": 1 }, { "command_buffer_ns": 18274541.65322706, "command_buffers": 1 }, { "command_buffer_ns": 19303000.008221716, "command_buffers": 1 }, { "command_buffer_ns": 17346375.03977865, "command_buffers": 1 }, { "command_buffer_ns": 17388333.33365619, "command_buffers": 1 }, { "command_buffer_ns": 17210749.96748939, "command_buffers": 1 }, { "command_buffer_ns": 17350083.333440125, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 281541.6664816439, "command_buffers": 1 }, { "command_buffer_ns": 272874.953225255, "command_buffers": 1 }, { "command_buffer_ns": 283708.3302438259, "command_buffers": 1 }, { "command_buffer_ns": 272916.62991046906, "command_buffers": 1 }, { "command_buffer_ns": 277291.692327708, "command_buffers": 1 }, { "command_buffer_ns": 272833.3347477019, "command_buffers": 1 }, { "command_buffer_ns": 284833.3097063005, "command_buffers": 1 }, { "command_buffer_ns": 274958.38003233075, "command_buffers": 1 }, { "command_buffer_ns": 278000.021353364, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 294.95052149286494, 303.4707033293671, 296.98242178710643, 285.53971333167283, 301.6093751284643, 271.0371099965414, 271.692708338378, 268.91796824202174, 271.09505208500195 ], "command_buffer_throughput_us_p50": 285.53971333167283, "command_buffer_throughput_us_p90": 301.98164076864487, "command_buffer_latency_us": [ 281.5416664816439, 272.874953225255, 283.7083302438259, 272.91662991046906, 277.291692327708, 272.8333347477019, 284.8333097063005, 274.95838003233075, 278.000021353364 ], "command_buffer_latency_us_p50": 277.291692327708, "command_buffer_latency_us_p90": 283.93332613632083 }, "throughput": [ { "compute_ns": 19036375.0, "compute_span_ns": 19036375.0, "command_buffer_ns": 19036375.044379383, "calibration_cpu_ns": 20733875.0, "calibration_gpu_ticks": 20733875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 19037667.0, "compute_span_ns": 19037667.0, "command_buffer_ns": 19037666.672375053, "calibration_cpu_ns": 20272833.0, "calibration_gpu_ticks": 20272833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21647333.0, "compute_span_ns": 21647333.0, "command_buffer_ns": 21647333.342116326, "calibration_cpu_ns": 23218500.0, "calibration_gpu_ticks": 23218500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21324000.0, "compute_span_ns": 21324000.0, "command_buffer_ns": 21324000.030290335, "calibration_cpu_ns": 24339083.0, "calibration_gpu_ticks": 24339083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 19720791.0, "compute_span_ns": 19720791.0, "command_buffer_ns": 19720750.04596263, "calibration_cpu_ns": 20794458.0, "calibration_gpu_ticks": 20794458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17314542.0, "compute_span_ns": 17314542.0, "command_buffer_ns": 17314583.354163915, "calibration_cpu_ns": 22541166.0, "calibration_gpu_ticks": 22541166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17275375.0, "compute_span_ns": 17275375.0, "command_buffer_ns": 17275375.02301857, "calibration_cpu_ns": 19394875.0, "calibration_gpu_ticks": 19394875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17365417.0, "compute_span_ns": 17365417.0, "command_buffer_ns": 17365458.36320147, "calibration_cpu_ns": 18474625.0, "calibration_gpu_ticks": 18474625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17227042.0, "compute_span_ns": 17227042.0, "command_buffer_ns": 17227041.6514948, "calibration_cpu_ns": 18917625.0, "calibration_gpu_ticks": 18917625.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 273333.0, "compute_span_ns": 273333.0, "command_buffer_ns": 273333.33855494857, "calibration_cpu_ns": 1103167.0, "calibration_gpu_ticks": 1103167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 276666.0, "compute_span_ns": 276666.0, "command_buffer_ns": 276624.98177960515, "calibration_cpu_ns": 881417.0, "calibration_gpu_ticks": 881417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 273209.0, "compute_span_ns": 273209.0, "command_buffer_ns": 273250.0433921814, "calibration_cpu_ns": 803375.0, "calibration_gpu_ticks": 803375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275333.0, "compute_span_ns": 275333.0, "command_buffer_ns": 275333.3537839353, "calibration_cpu_ns": 1282209.0, "calibration_gpu_ticks": 1282209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 273708.0, "compute_span_ns": 273708.0, "command_buffer_ns": 273666.693829, "calibration_cpu_ns": 836750.0, "calibration_gpu_ticks": 836750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 278417.0, "compute_span_ns": 278417.0, "command_buffer_ns": 278416.6717901826, "calibration_cpu_ns": 842791.0, "calibration_gpu_ticks": 842791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 273334.0, "compute_span_ns": 273334.0, "command_buffer_ns": 273375.0152401626, "calibration_cpu_ns": 1152125.0, "calibration_gpu_ticks": 1152125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 277708.0, "compute_span_ns": 277708.0, "command_buffer_ns": 277708.3427645266, "calibration_cpu_ns": 1132667.0, "calibration_gpu_ticks": 1132667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 275417.0, "compute_span_ns": 275417.0, "command_buffer_ns": 275416.7071543634, "calibration_cpu_ns": 954542.0, "calibration_gpu_ticks": 954542.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 297.443359375, 297.463546875, 338.239578125, 333.1875, 308.137359375, 270.53971875, 269.927734375, 271.334640625, 269.17253125 ], "compute_throughput_us_p50": 297.443359375, "compute_throughput_us_p90": 334.197915625, "command_buffer_throughput_us": [ 297.44336006842786, 297.4635417558602, 338.2395834705676, 333.1875004732865, 308.1367194681661, 270.54036490881117, 269.92773473466514, 271.33528692502296, 269.17252580460627 ], "command_buffer_throughput_us_p50": 297.44336006842786, "command_buffer_throughput_us_p90": 334.1979170727427, "compute_latency_us": [ 273.333, 276.666, 273.209, 275.333, 273.708, 278.417, 273.334, 277.708, 275.417 ], "compute_latency_us_p50": 275.333, "compute_latency_us_p90": 277.8498, "command_buffer_latency_us": [ 273.33333855494857, 276.62498177960515, 273.2500433921814, 275.3333537839353, 273.666693829, 278.4166717901826, 273.3750152401626, 277.7083427645266, 275.4167071543634 ], "command_buffer_latency_us_p50": 275.3333537839353, "command_buffer_latency_us_p90": 277.8500085696578, "counter_control_throughput_ratio": 1.008451717809976, "counter_control_latency_ratio": 0.9907075035950557 }, "download_ms": 1.572875, "allocation_upload_ms": 1.228208, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-0h981zxf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "2f415c8199d32d1b999ce6dae6674c9bc8bbdf9b0adda641a6d2aee649468028", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 24.174, "capture_ms": 0.099083, "compile_ms": 92.074042, "allocation_upload_ms": 7.653125, "cold_call_ms": 8.604333, "warmup_ms": 201.008458, "download_ms": 6.641916, "repetitions": 412, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 72.1553398058, 72.8881456311, 72.1030533981, 72.1999393204, 72.0639174757, 72.256776699, 71.7942961165, 72.1989296117, 72.9864490291 ], "latency_us": [ 384.208, 380.209, 330.167, 335.041, 326.916, 284.958, 292.667, 300.625, 291.541 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4603958, "compute_span_ns": 4603958, "calibration_cpu_ns": 5187875, "calibration_gpu_ticks": 5187875, "compute_passes": 1, "command_buffer_ns": 4603958.33943, "command_buffers": 2 }, { "compute_ns": 4641000, "compute_span_ns": 4641000, "calibration_cpu_ns": 5466500, "calibration_gpu_ticks": 5466500, "compute_passes": 1, "command_buffer_ns": 4641041.6835, "command_buffers": 2 }, { "compute_ns": 4690625, "compute_span_ns": 4690625, "calibration_cpu_ns": 5460958, "calibration_gpu_ticks": 5460958, "compute_passes": 1, "command_buffer_ns": 4690708.3015, "command_buffers": 2 }, { "compute_ns": 4661000, "compute_span_ns": 4661000, "calibration_cpu_ns": 5579125, "calibration_gpu_ticks": 5579125, "compute_passes": 1, "command_buffer_ns": 4661083.33785, "command_buffers": 2 }, { "compute_ns": 4687208, "compute_span_ns": 4687208, "calibration_cpu_ns": 5179083, "calibration_gpu_ticks": 5179083, "compute_passes": 1, "command_buffer_ns": 4687166.65637, "command_buffers": 2 }, { "compute_ns": 4641791, "compute_span_ns": 4641791, "calibration_cpu_ns": 5181667, "calibration_gpu_ticks": 5181667, "compute_passes": 1, "command_buffer_ns": 4641791.68921, "command_buffers": 2 }, { "compute_ns": 4765792, "compute_span_ns": 4765792, "calibration_cpu_ns": 5790250, "calibration_gpu_ticks": 5790250, "compute_passes": 1, "command_buffer_ns": 4765875.05545, "command_buffers": 2 }, { "compute_ns": 4880500, "compute_span_ns": 4880500, "calibration_cpu_ns": 5745959, "calibration_gpu_ticks": 5745959, "compute_passes": 1, "command_buffer_ns": 4880499.96784, "command_buffers": 2 }, { "compute_ns": 4973209, "compute_span_ns": 4973209, "calibration_cpu_ns": 5850583, "calibration_gpu_ticks": 5850583, "compute_passes": 1, "command_buffer_ns": 4973291.65228, "command_buffers": 2 } ], "latency": [ { "compute_ns": 84291, "compute_span_ns": 84291, "calibration_cpu_ns": 600333, "calibration_gpu_ticks": 600333, "compute_passes": 1, "command_buffer_ns": 84250.0012368, "command_buffers": 2 }, { "compute_ns": 73875, "compute_span_ns": 73875, "calibration_cpu_ns": 547125, "calibration_gpu_ticks": 547125, "compute_passes": 1, "command_buffer_ns": 73999.9813959, "command_buffers": 2 }, { "compute_ns": 183750, "compute_span_ns": 183750, "calibration_cpu_ns": 1063584, "calibration_gpu_ticks": 1063584, "compute_passes": 1, "command_buffer_ns": 183833.35555, "command_buffers": 2 }, { "compute_ns": 73167, "compute_span_ns": 73167, "calibration_cpu_ns": 568458, "calibration_gpu_ticks": 568458, "compute_passes": 1, "command_buffer_ns": 73208.2989998, "command_buffers": 2 }, { "compute_ns": 94042, "compute_span_ns": 94042, "calibration_cpu_ns": 610041, "calibration_gpu_ticks": 610041, "compute_passes": 1, "command_buffer_ns": 94166.7240113, "command_buffers": 2 }, { "compute_ns": 78459, "compute_span_ns": 78459, "calibration_cpu_ns": 641625, "calibration_gpu_ticks": 641625, "compute_passes": 1, "command_buffer_ns": 78541.6923463, "command_buffers": 2 }, { "compute_ns": 86083, "compute_span_ns": 86083, "calibration_cpu_ns": 609625, "calibration_gpu_ticks": 609625, "compute_passes": 1, "command_buffer_ns": 86166.6630954, "command_buffers": 2 }, { "compute_ns": 104875, "compute_span_ns": 104875, "calibration_cpu_ns": 713667, "calibration_gpu_ticks": 713667, "compute_passes": 1, "command_buffer_ns": 104958.366137, "command_buffers": 2 }, { "compute_ns": 124375, "compute_span_ns": 124375, "calibration_cpu_ns": 678625, "calibration_gpu_ticks": 678625, "compute_passes": 1, "command_buffer_ns": 124500.016682, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4748166.65519, "command_buffers": 2 }, { "command_buffer_ns": 4613833.2691, "command_buffers": 2 }, { "command_buffer_ns": 4640458.32632, "command_buffers": 2 }, { "command_buffer_ns": 4641500.01062, "command_buffers": 2 }, { "command_buffer_ns": 4623916.64041, "command_buffers": 2 }, { "command_buffer_ns": 4666541.64484, "command_buffers": 2 }, { "command_buffer_ns": 4624208.37721, "command_buffers": 2 }, { "command_buffer_ns": 5079291.5863, "command_buffers": 2 }, { "command_buffer_ns": 4909458.27914, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 243708.316702, "command_buffers": 2 }, { "command_buffer_ns": 73583.3309591, "command_buffers": 2 }, { "command_buffer_ns": 85291.6273288, "command_buffers": 2 }, { "command_buffer_ns": 86083.3097249, "command_buffers": 2 }, { "command_buffer_ns": 124333.309941, "command_buffers": 2 }, { "command_buffer_ns": 105625.018477, "command_buffers": 2 }, { "command_buffer_ns": 133416.673634, "command_buffers": 2 }, { "command_buffer_ns": 158625.014592, "command_buffers": 2 }, { "command_buffer_ns": 94083.3706409, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 74.19010398734376, 72.09114482968751, 72.50716134875, 72.5234376659375, 72.24869750640626, 72.914713200625, 72.25325589390624, 79.3639310359375, 76.7102856115625 ], "command_buffer_throughput_us_p50": 72.5234376659375, "command_buffer_throughput_us_p90": 77.2410146964375, "command_buffer_latency_us": [ 243.70831670200002, 73.5833309591, 85.29162732879999, 86.0833097249, 124.333309941, 105.62501847700001, 133.416673634, 158.62501459199999, 94.0833706409 ], "command_buffer_latency_us_p50": 105.62501847700001, "command_buffer_latency_us_p90": 175.64167501400001 }, "compute_throughput_us": [ 71.93684375, 72.515625, 73.291015625, 72.828125, 73.237625, 72.527984375, 74.4655, 76.2578125, 77.706390625 ], "compute_throughput_us_p50": 73.237625, "compute_throughput_us_p90": 76.547528125, "command_buffer_throughput_us": [ 71.93684905359375, 72.51627630468751, 73.2923172109375, 72.82942715390625, 73.23697900578125, 72.52799514390625, 74.46679774140625, 76.2578119975, 77.707682066875 ], "command_buffer_throughput_us_p50": 73.23697900578125, "command_buffer_throughput_us_p90": 76.547786011375, "compute_latency_us": [ 84.291, 73.875, 183.75, 73.167, 94.042, 78.459, 86.083, 104.875, 124.375 ], "compute_latency_us_p50": 86.083, "compute_latency_us_p90": 136.25, "command_buffer_latency_us": [ 84.2500012368, 73.99998139590001, 183.83335555000002, 73.2082989998, 94.1667240113, 78.5416923463, 86.1666630954, 104.958366137, 124.500016682 ], "command_buffer_latency_us_p50": 86.1666630954, "command_buffer_latency_us_p90": 136.36668445560002, "counter_control_throughput_ratio": 1.0058971386292221, "counter_control_latency_ratio": 0.7573732578661746 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 72.1989296117, "throughput_us_p90": 72.9078063107, "latency_us_p50": 326.916, "latency_us_p90": 381.0088 }, "slowdown": 0.234025097119156, "device_compute_slowdown": 0.24622376896861928, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-nqtx6xpj/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7d02ca31ce0660bb7b8cf2d4b7c68e2de8afe58cfb625e16c6aa93173ce8f234", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 28.719792, "capture_ms": 0.070666, "compile_ms": 63.486916, "allocation_upload_ms": 6.097458, "cold_call_ms": 7.724958, "warmup_ms": 200.068292, "download_ms": 3.268417, "repetitions": 346, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 76.5275780347, 75.9432803468, 76.1164508671, 76.1728092486, 76.4517109827, 77.1145231214, 76.7337427746, 76.7862485549, 76.8697023121 ], "latency_us": [ 324.5, 293.625, 286.75, 306.917, 294.75, 271.75, 279.042, 297.042, 277.042 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4826833, "compute_span_ns": 4826833, "calibration_cpu_ns": 5470958, "calibration_gpu_ticks": 5470958, "compute_passes": 1, "command_buffer_ns": 4826833.31938, "command_buffers": 2 }, { "compute_ns": 4815583, "compute_span_ns": 4815583, "calibration_cpu_ns": 5288416, "calibration_gpu_ticks": 5288416, "compute_passes": 1, "command_buffer_ns": 4815625.02682, "command_buffers": 2 }, { "compute_ns": 4848083, "compute_span_ns": 4848083, "calibration_cpu_ns": 5318167, "calibration_gpu_ticks": 5318167, "compute_passes": 1, "command_buffer_ns": 4848083.30657, "command_buffers": 2 }, { "compute_ns": 4871417, "compute_span_ns": 4871417, "calibration_cpu_ns": 5368834, "calibration_gpu_ticks": 5368834, "compute_passes": 1, "command_buffer_ns": 4871458.33904, "command_buffers": 2 }, { "compute_ns": 4897875, "compute_span_ns": 4897875, "calibration_cpu_ns": 5393416, "calibration_gpu_ticks": 5393416, "compute_passes": 1, "command_buffer_ns": 4897916.68952, "command_buffers": 2 }, { "compute_ns": 4829959, "compute_span_ns": 4829959, "calibration_cpu_ns": 5341167, "calibration_gpu_ticks": 5341167, "compute_passes": 1, "command_buffer_ns": 4830083.28592, "command_buffers": 2 }, { "compute_ns": 4841792, "compute_span_ns": 4841792, "calibration_cpu_ns": 5380875, "calibration_gpu_ticks": 5380875, "compute_passes": 1, "command_buffer_ns": 4841791.6405, "command_buffers": 2 }, { "compute_ns": 4846417, "compute_span_ns": 4846417, "calibration_cpu_ns": 5365875, "calibration_gpu_ticks": 5365875, "compute_passes": 1, "command_buffer_ns": 4846541.67667, "command_buffers": 2 }, { "compute_ns": 4858625, "compute_span_ns": 4858625, "calibration_cpu_ns": 5298083, "calibration_gpu_ticks": 5298083, "compute_passes": 1, "command_buffer_ns": 4858666.68168, "command_buffers": 2 } ], "latency": [ { "compute_ns": 75875, "compute_span_ns": 75875, "calibration_cpu_ns": 473792, "calibration_gpu_ticks": 473792, "compute_passes": 1, "command_buffer_ns": 75916.7014621, "command_buffers": 2 }, { "compute_ns": 76584, "compute_span_ns": 76584, "calibration_cpu_ns": 497292, "calibration_gpu_ticks": 497292, "compute_passes": 1, "command_buffer_ns": 76708.3256505, "command_buffers": 2 }, { "compute_ns": 76500, "compute_span_ns": 76500, "calibration_cpu_ns": 453250, "calibration_gpu_ticks": 453250, "compute_passes": 1, "command_buffer_ns": 76583.3538026, "command_buffers": 2 }, { "compute_ns": 77875, "compute_span_ns": 77875, "calibration_cpu_ns": 457709, "calibration_gpu_ticks": 457709, "compute_passes": 1, "command_buffer_ns": 77958.3351687, "command_buffers": 2 }, { "compute_ns": 83959, "compute_span_ns": 83959, "calibration_cpu_ns": 471917, "calibration_gpu_ticks": 471917, "compute_passes": 1, "command_buffer_ns": 83999.9993332, "command_buffers": 2 }, { "compute_ns": 76791, "compute_span_ns": 76791, "calibration_cpu_ns": 430625, "calibration_gpu_ticks": 430625, "compute_passes": 1, "command_buffer_ns": 76791.679021, "command_buffers": 2 }, { "compute_ns": 77916, "compute_span_ns": 77916, "calibration_cpu_ns": 446417, "calibration_gpu_ticks": 446417, "compute_passes": 1, "command_buffer_ns": 77958.3351687, "command_buffers": 2 }, { "compute_ns": 75625, "compute_span_ns": 75625, "calibration_cpu_ns": 453375, "calibration_gpu_ticks": 453375, "compute_passes": 1, "command_buffer_ns": 75625.0228733, "command_buffers": 2 }, { "compute_ns": 90042, "compute_span_ns": 90042, "calibration_cpu_ns": 504917, "calibration_gpu_ticks": 504917, "compute_passes": 1, "command_buffer_ns": 90208.3702385, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4923833.3595, "command_buffers": 2 }, { "command_buffer_ns": 4867208.36489, "command_buffers": 2 }, { "command_buffer_ns": 4843750.03725, "command_buffers": 2 }, { "command_buffer_ns": 4900083.35328, "command_buffers": 2 }, { "command_buffer_ns": 4843791.65573, "command_buffers": 2 }, { "command_buffer_ns": 4880708.35127, "command_buffers": 2 }, { "command_buffer_ns": 4849124.99087, "command_buffers": 2 }, { "command_buffer_ns": 4844249.98285, "command_buffers": 2 }, { "command_buffer_ns": 4877208.26641, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 77958.3933763, "command_buffers": 2 }, { "command_buffer_ns": 76208.380051, "command_buffers": 2 }, { "command_buffer_ns": 76708.3256505, "command_buffers": 2 }, { "command_buffer_ns": 77833.305113, "command_buffers": 2 }, { "command_buffer_ns": 83499.9955259, "command_buffers": 2 }, { "command_buffer_ns": 77874.9817982, "command_buffers": 2 }, { "command_buffer_ns": 76750.0023358, "command_buffers": 2 }, { "command_buffer_ns": 77250.0643507, "command_buffers": 2 }, { "command_buffer_ns": 77249.9479353, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 76.9348962421875, 76.05013070140625, 75.68359433203125, 76.56380239500001, 75.68424462078124, 76.26106798859375, 75.76757798234375, 75.69140598203126, 76.20637916265625 ], "command_buffer_throughput_us_p50": 76.05013070140625, "command_buffer_throughput_us_p90": 76.6380211644375, "command_buffer_latency_us": [ 77.9583933763, 76.208380051, 76.7083256505, 77.833305113, 83.4999955259, 77.8749817982, 76.7500023358, 77.2500643507, 77.2499479353 ], "command_buffer_latency_us_p50": 77.2500643507, "command_buffer_latency_us_p90": 79.06671380622 }, "compute_throughput_us": [ 75.419265625, 75.243484375, 75.751296875, 76.115890625, 76.529296875, 75.468109375, 75.653, 75.725265625, 75.916015625 ], "compute_throughput_us_p50": 75.725265625, "compute_throughput_us_p90": 76.198571875, "command_buffer_throughput_us": [ 75.4192706153125, 75.2441410440625, 75.75130166515625, 76.1165365475, 76.52994827375, 75.4700513425, 75.6529943828125, 75.72721369796875, 75.91666690125001 ], "command_buffer_throughput_us_p50": 75.72721369796875, "command_buffer_throughput_us_p90": 76.19921889275, "compute_latency_us": [ 75.875, 76.584, 76.5, 77.875, 83.959, 76.791, 77.916, 75.625, 90.042 ], "compute_latency_us_p50": 76.791, "compute_latency_us_p90": 85.1756, "command_buffer_latency_us": [ 75.9167014621, 76.7083256505, 76.5833538026, 77.95833516869999, 83.9999993332, 76.79167902100001, 77.95833516869999, 75.6250228733, 90.20837023850001 ], "command_buffer_latency_us_p50": 76.79167902100001, "command_buffer_latency_us_p90": 85.24167351426, "counter_control_throughput_ratio": 0.9961983200804243, "counter_control_latency_ratio": 1.0016063824543808 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 76.5275780347, "throughput_us_p90": 76.91866647396, "latency_us_p50": 293.625, "latency_us_p90": 310.43359999999996 }, "torch": { "cold_call_ms": 6.961834, "warmup_ms": 200.727375, "repetitions": 143, "throughput_us": [ 208.99388111888112, 205.15239160839158, 206.05506993006992, 205.66753846153844, 204.58041958041957, 204.93094405594405, 204.78904195804196, 206.36072027972028, 207.05565734265733 ], "latency_us": [ 599.7909999999999, 466.04200000000003, 466.667, 456.875, 442.125, 446.20799999999997, 453.542, 462.541, 447.375 ], "throughput_us_p50": 205.66753846153844, "throughput_us_p90": 207.4433020979021, "latency_us_p50": 456.875, "latency_us_p90": 493.29179999999997, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 12826874.968595803, "command_buffers": 1 }, { "command_buffer_ns": 12782083.300407976, "command_buffers": 1 }, { "command_buffer_ns": 12963666.69703275, "command_buffers": 1 }, { "command_buffer_ns": 12928500.01482293, "command_buffers": 1 }, { "command_buffer_ns": 12893583.334516734, "command_buffers": 1 }, { "command_buffer_ns": 12869958.358351141, "command_buffers": 1 }, { "command_buffer_ns": 12958916.660863906, "command_buffers": 1 }, { "command_buffer_ns": 13095083.297230303, "command_buffers": 1 }, { "command_buffer_ns": 13199749.984778464, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 200249.9531954527, "command_buffers": 1 }, { "command_buffer_ns": 202249.96842443943, "command_buffers": 1 }, { "command_buffer_ns": 200583.30846950412, "command_buffers": 1 }, { "command_buffer_ns": 199750.00759586692, "command_buffers": 1 }, { "command_buffer_ns": 200500.01330673695, "command_buffers": 1 }, { "command_buffer_ns": 201541.63939878345, "command_buffers": 1 }, { "command_buffer_ns": 238749.95531514287, "command_buffers": 1 }, { "command_buffer_ns": 201291.6374951601, "command_buffers": 1 }, { "command_buffer_ns": 199541.68237745762, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 200.41992138430942, 199.72005156887462, 202.5572921411367, 202.00781273160828, 201.46223960182397, 201.09309934923658, 202.48307282599853, 204.6106765192235, 206.2460935121635 ], "command_buffer_throughput_us_p50": 202.00781273160828, "command_buffer_throughput_us_p90": 204.9377599178115, "command_buffer_latency_us": [ 200.2499531954527, 202.24996842443943, 200.58330846950412, 199.75000759586692, 200.50001330673695, 201.54163939878345, 238.74995531514287, 201.2916374951601, 199.54168237745762 ], "command_buffer_latency_us_p50": 200.58330846950412, "command_buffer_latency_us_p90": 209.54996580258012 }, "throughput": [ { "compute_ns": 12919875.0, "compute_span_ns": 12919875.0, "command_buffer_ns": 12919875.036459416, "calibration_cpu_ns": 13759333.0, "calibration_gpu_ticks": 13759333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12866667.0, "compute_span_ns": 12866667.0, "command_buffer_ns": 12866666.656918824, "calibration_cpu_ns": 13742042.0, "calibration_gpu_ticks": 13742042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12885833.0, "compute_span_ns": 12885833.0, "command_buffer_ns": 12885833.333712071, "calibration_cpu_ns": 14035416.0, "calibration_gpu_ticks": 14035416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12894708.0, "compute_span_ns": 12894708.0, "command_buffer_ns": 12894666.637293994, "calibration_cpu_ns": 14023500.0, "calibration_gpu_ticks": 14023500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12965250.0, "compute_span_ns": 12965250.0, "command_buffer_ns": 12965250.003617257, "calibration_cpu_ns": 13865292.0, "calibration_gpu_ticks": 13865292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12979625.0, "compute_span_ns": 12979625.0, "command_buffer_ns": 12979625.025764108, "calibration_cpu_ns": 13753709.0, "calibration_gpu_ticks": 13753709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13010417.0, "compute_span_ns": 13010417.0, "command_buffer_ns": 13010458.322241902, "calibration_cpu_ns": 14960125.0, "calibration_gpu_ticks": 14960125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12970583.0, "compute_span_ns": 12970583.0, "command_buffer_ns": 12970583.338756114, "calibration_cpu_ns": 15067708.0, "calibration_gpu_ticks": 15067708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13159208.0, "compute_span_ns": 13159208.0, "command_buffer_ns": 13159208.348952234, "calibration_cpu_ns": 14408000.0, "calibration_gpu_ticks": 14408000.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 204208.0, "compute_span_ns": 204208.0, "command_buffer_ns": 204166.688490659, "calibration_cpu_ns": 805750.0, "calibration_gpu_ticks": 805750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200500.0, "compute_span_ns": 200500.0, "command_buffer_ns": 200500.01330673695, "calibration_cpu_ns": 755333.0, "calibration_gpu_ticks": 755333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201792.0, "compute_span_ns": 201792.0, "command_buffer_ns": 201833.37619528174, "calibration_cpu_ns": 736292.0, "calibration_gpu_ticks": 736292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200500.0, "compute_span_ns": 200500.0, "command_buffer_ns": 200500.01330673695, "calibration_cpu_ns": 637125.0, "calibration_gpu_ticks": 637125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202750.0, "compute_span_ns": 202750.0, "command_buffer_ns": 202750.03043934703, "calibration_cpu_ns": 808250.0, "calibration_gpu_ticks": 808250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202208.0, "compute_span_ns": 202208.0, "command_buffer_ns": 202166.67326167226, "calibration_cpu_ns": 755667.0, "calibration_gpu_ticks": 755667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200833.0, "compute_span_ns": 200833.0, "command_buffer_ns": 200833.31037312746, "calibration_cpu_ns": 881708.0, "calibration_gpu_ticks": 881708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200875.0, "compute_span_ns": 200875.0, "command_buffer_ns": 200875.04526600242, "calibration_cpu_ns": 972416.0, "calibration_gpu_ticks": 972416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 238209.0, "compute_span_ns": 238209.0, "command_buffer_ns": 238249.95150789618, "calibration_cpu_ns": 871417.0, "calibration_gpu_ticks": 871417.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 201.873046875, 201.041671875, 201.341140625, 201.4798125, 202.58203125, 202.806640625, 203.287765625, 202.665359375, 205.612625 ], "compute_throughput_us_p50": 202.58203125, "compute_throughput_us_p90": 203.7527375, "command_buffer_throughput_us": [ 201.87304744467838, 201.04166651435662, 201.3411458392511, 201.47916620771866, 202.58203130651964, 202.80664102756418, 203.28841128502972, 202.66536466806429, 205.61263045237865 ], "command_buffer_throughput_us_p50": 202.58203130651964, "command_buffer_throughput_us_p90": 203.7532551184995, "compute_latency_us": [ 204.208, 200.5, 201.792, 200.5, 202.75, 202.208, 200.833, 200.875, 238.209 ], "compute_latency_us_p50": 201.792, "compute_latency_us_p90": 211.00820000000002, "command_buffer_latency_us": [ 204.166688490659, 200.50001330673695, 201.83337619528174, 200.50001330673695, 202.75003043934703, 202.16667326167226, 200.83331037312746, 200.87504526600242, 238.24995150789618 ], "command_buffer_latency_us_p50": 201.83337619528174, "command_buffer_latency_us_p90": 210.98334109410644, "counter_control_throughput_ratio": 1.00397731251206, "counter_control_latency_ratio": 1.003754721814016 }, "download_ms": 1.02575, "allocation_upload_ms": 16.877208, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3720936157798733, "device_compute_slowdown": 0.3738005051965832, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.52175, "warmup_ms": 201.355, "repetitions": 142, "throughput_us": [ 210.97241549295774, 213.58128169014083, 213.89935915492958, 211.152, 212.60064084507044, 211.5314014084507, 212.23973239436617, 214.64260563380282, 213.11326056338027 ], "latency_us": [ 502.45799999999997, 415.95799999999997, 472.04200000000003, 486.584, 535.75, 558.5, 600.125, 491.70799999999997, 417.5 ], "throughput_us_p50": 212.60064084507044, "throughput_us_p90": 214.04800845070423, "latency_us_p50": 491.70799999999997, "latency_us_p90": 566.825, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 13414458.313491195, "command_buffers": 1 }, { "command_buffer_ns": 13362166.70460999, "command_buffers": 1 }, { "command_buffer_ns": 13298375.008162111, "command_buffers": 1 }, { "command_buffer_ns": 13136083.318386227, "command_buffers": 1 }, { "command_buffer_ns": 13188874.98928234, "command_buffers": 1 }, { "command_buffer_ns": 13443375.006318092, "command_buffers": 1 }, { "command_buffer_ns": 13572333.380579948, "command_buffers": 1 }, { "command_buffer_ns": 13164458.330720663, "command_buffers": 1 }, { "command_buffer_ns": 13351458.2994394, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 200791.69189557433, "command_buffers": 1 }, { "command_buffer_ns": 200208.33471789956, "command_buffers": 1 }, { "command_buffer_ns": 200999.95890632272, "command_buffers": 1 }, { "command_buffer_ns": 201333.37238803506, "command_buffers": 1 }, { "command_buffer_ns": 200374.98325109482, "command_buffers": 1 }, { "command_buffer_ns": 201458.34423601627, "command_buffers": 1 }, { "command_buffer_ns": 264833.33203941584, "command_buffers": 1 }, { "command_buffer_ns": 201500.02092123032, "command_buffers": 1 }, { "command_buffer_ns": 201375.0490732491, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 209.60091114829993, 208.7838547595311, 207.78710950253299, 205.2513018497848, 206.07617170753656, 210.0527344737202, 212.0677090715617, 205.69466141751036, 208.61653592874063 ], "command_buffer_throughput_us_p50": 208.61653592874063, "command_buffer_throughput_us_p90": 210.4557293932885, "command_buffer_latency_us": [ 200.79169189557433, 200.20833471789956, 200.99995890632272, 201.33337238803506, 200.37498325109482, 201.45834423601627, 264.83333203941584, 201.50002092123032, 201.3750490732491 ], "command_buffer_latency_us_p50": 201.33337238803506, "command_buffer_latency_us_p90": 214.16668314486742 }, "throughput": [ { "compute_ns": 13328709.0, "compute_span_ns": 13328709.0, "command_buffer_ns": 13328749.977517873, "calibration_cpu_ns": 14195917.0, "calibration_gpu_ticks": 14195917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13207916.0, "compute_span_ns": 13207916.0, "command_buffer_ns": 13207875.017542392, "calibration_cpu_ns": 14015583.0, "calibration_gpu_ticks": 14015583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13260500.0, "compute_span_ns": 13260500.0, "command_buffer_ns": 13260500.03990531, "calibration_cpu_ns": 14188875.0, "calibration_gpu_ticks": 14188875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13158916.0, "compute_span_ns": 13158916.0, "command_buffer_ns": 13158874.993678182, "calibration_cpu_ns": 14040375.0, "calibration_gpu_ticks": 14040375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13618541.0, "compute_span_ns": 13618541.0, "command_buffer_ns": 13618500.030133873, "calibration_cpu_ns": 14688792.0, "calibration_gpu_ticks": 14688792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13544750.0, "compute_span_ns": 13544750.0, "command_buffer_ns": 13544749.992433935, "calibration_cpu_ns": 14674792.0, "calibration_gpu_ticks": 14674792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13481375.0, "compute_span_ns": 13481375.0, "command_buffer_ns": 13481375.004630536, "calibration_cpu_ns": 14515333.0, "calibration_gpu_ticks": 14515333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13491666.0, "compute_span_ns": 13491666.0, "command_buffer_ns": 13491625.024471432, "calibration_cpu_ns": 14503208.0, "calibration_gpu_ticks": 14503208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13198250.0, "compute_span_ns": 13198250.0, "command_buffer_ns": 13198250.031564385, "calibration_cpu_ns": 14118333.0, "calibration_gpu_ticks": 14118333.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 200792.0, "compute_span_ns": 200792.0, "command_buffer_ns": 200833.31037312746, "calibration_cpu_ns": 615250.0, "calibration_gpu_ticks": 615250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201500.0, "compute_span_ns": 201500.0, "command_buffer_ns": 201500.02092123032, "calibration_cpu_ns": 620417.0, "calibration_gpu_ticks": 620417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 264584.0, "compute_span_ns": 264584.0, "command_buffer_ns": 264625.00682100654, "calibration_cpu_ns": 806125.0, "calibration_gpu_ticks": 806125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201291.0, "compute_span_ns": 201291.0, "command_buffer_ns": 201250.01901760697, "calibration_cpu_ns": 640750.0, "calibration_gpu_ticks": 640750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201417.0, "compute_span_ns": 201417.0, "command_buffer_ns": 201458.34423601627, "calibration_cpu_ns": 639125.0, "calibration_gpu_ticks": 639125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199500.0, "compute_span_ns": 199500.0, "command_buffer_ns": 199500.00569224358, "calibration_cpu_ns": 625833.0, "calibration_gpu_ticks": 625833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201958.0, "compute_span_ns": 201958.0, "command_buffer_ns": 201916.67135804892, "calibration_cpu_ns": 642834.0, "calibration_gpu_ticks": 642834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199917.0, "compute_span_ns": 199917.0, "command_buffer_ns": 199958.33281427622, "calibration_cpu_ns": 614209.0, "calibration_gpu_ticks": 614209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199917.0, "compute_span_ns": 199917.0, "command_buffer_ns": 199958.33281427622, "calibration_cpu_ns": 602084.0, "calibration_gpu_ticks": 602084.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 208.261078125, 206.3736875, 207.1953125, 205.6080625, 212.789703125, 211.63671875, 210.646484375, 210.80728125, 206.22265625 ], "compute_throughput_us_p50": 208.261078125, "compute_throughput_us_p90": 211.867315625, "command_buffer_throughput_us": [ 208.26171839871677, 206.37304714909988, 207.19531312352046, 205.6074217762216, 212.78906297084177, 211.63671863178024, 210.64648444735212, 210.80664100736612, 206.2226567431935 ], "command_buffer_throughput_us_p50": 208.26171839871677, "command_buffer_throughput_us_p90": 211.86718749959255, "compute_latency_us": [ 200.792, 201.5, 264.584, 201.291, 201.417, 199.5, 201.958, 199.917, 199.917 ], "compute_latency_us_p50": 201.291, "compute_latency_us_p90": 214.4832, "command_buffer_latency_us": [ 200.83331037312746, 201.50002092123032, 264.62500682100654, 201.25001901760697, 201.45834423601627, 199.50000569224358, 201.91667135804892, 199.95833281427622, 199.95833281427622 ], "command_buffer_latency_us_p50": 201.25001901760697, "command_buffer_latency_us_p90": 214.45833845064044, "counter_control_throughput_ratio": 0.997151910046637, "counter_control_latency_ratio": 0.9995859932735471 }, "download_ms": 0.623292, "allocation_upload_ms": 1.505625, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-vb4ocm41/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "6caff4b0a979c3104fea9f23f0d5430392f6f4e7d743dab03e9fdb6de1428bf3", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.162875, "capture_ms": 0.078834, "compile_ms": 58.465959, "allocation_upload_ms": 5.82275, "cold_call_ms": 5.007292, "warmup_ms": 201.097459, "download_ms": 3.583083, "repetitions": 454, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 61.8627026432, 62.4648502203, 61.5594713656, 62.7985484581, 62.7799185022, 63.0672731278, 62.016154185, 62.1504207048, 62.158222467 ], "latency_us": [ 351.958, 291.25, 304.375, 400.959, 302.375, 298.917, 286.083, 300, 308.834 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3939042, "compute_span_ns": 3939042, "calibration_cpu_ns": 4454458, "calibration_gpu_ticks": 4454458, "compute_passes": 1, "command_buffer_ns": 3939166.72328, "command_buffers": 2 }, { "compute_ns": 3964917, "compute_span_ns": 3964917, "calibration_cpu_ns": 4458084, "calibration_gpu_ticks": 4458084, "compute_passes": 1, "command_buffer_ns": 3965083.33506, "command_buffers": 2 }, { "compute_ns": 3927625, "compute_span_ns": 3927625, "calibration_cpu_ns": 4442375, "calibration_gpu_ticks": 4442375, "compute_passes": 1, "command_buffer_ns": 3927708.37061, "command_buffers": 2 }, { "compute_ns": 4018125, "compute_span_ns": 4018125, "calibration_cpu_ns": 4456041, "calibration_gpu_ticks": 4456041, "compute_passes": 1, "command_buffer_ns": 4018249.97971, "command_buffers": 2 }, { "compute_ns": 3965958, "compute_span_ns": 3965958, "calibration_cpu_ns": 4439125, "calibration_gpu_ticks": 4439125, "compute_passes": 1, "command_buffer_ns": 3965958.31262, "command_buffers": 2 }, { "compute_ns": 3994541, "compute_span_ns": 3994541, "calibration_cpu_ns": 4475334, "calibration_gpu_ticks": 4475334, "compute_passes": 1, "command_buffer_ns": 3994500.03169, "command_buffers": 2 }, { "compute_ns": 3971750, "compute_span_ns": 3971750, "calibration_cpu_ns": 4426250, "calibration_gpu_ticks": 4426250, "compute_passes": 1, "command_buffer_ns": 3971750.03309, "command_buffers": 2 }, { "compute_ns": 3962417, "compute_span_ns": 3962417, "calibration_cpu_ns": 4437417, "calibration_gpu_ticks": 4437417, "compute_passes": 1, "command_buffer_ns": 3962458.28597, "command_buffers": 2 }, { "compute_ns": 3967792, "compute_span_ns": 3967792, "calibration_cpu_ns": 4520709, "calibration_gpu_ticks": 4520709, "compute_passes": 1, "command_buffer_ns": 3967875.03269, "command_buffers": 2 } ], "latency": [ { "compute_ns": 75583, "compute_span_ns": 75583, "calibration_cpu_ns": 568458, "calibration_gpu_ticks": 568458, "compute_passes": 1, "command_buffer_ns": 75583.3461881, "command_buffers": 2 }, { "compute_ns": 71334, "compute_span_ns": 71334, "calibration_cpu_ns": 544958, "calibration_gpu_ticks": 544958, "compute_passes": 1, "command_buffer_ns": 71374.9905117, "command_buffers": 2 }, { "compute_ns": 63584, "compute_span_ns": 63584, "calibration_cpu_ns": 472208, "calibration_gpu_ticks": 472208, "compute_passes": 1, "command_buffer_ns": 63666.6663922, "command_buffers": 2 }, { "compute_ns": 64459, "compute_span_ns": 64459, "calibration_cpu_ns": 489625, "calibration_gpu_ticks": 489625, "compute_passes": 1, "command_buffer_ns": 64500.0254735, "command_buffers": 2 }, { "compute_ns": 78458, "compute_span_ns": 78458, "calibration_cpu_ns": 578459, "calibration_gpu_ticks": 578459, "compute_passes": 1, "command_buffer_ns": 78458.3389759, "command_buffers": 2 }, { "compute_ns": 70125, "compute_span_ns": 70125, "calibration_cpu_ns": 564000, "calibration_gpu_ticks": 564000, "compute_passes": 1, "command_buffer_ns": 70208.334364, "command_buffers": 2 }, { "compute_ns": 63375, "compute_span_ns": 63375, "calibration_cpu_ns": 480917, "calibration_gpu_ticks": 480917, "compute_passes": 1, "command_buffer_ns": 63458.3411738, "command_buffers": 2 }, { "compute_ns": 64416, "compute_span_ns": 64416, "calibration_cpu_ns": 583500, "calibration_gpu_ticks": 583500, "compute_passes": 1, "command_buffer_ns": 64416.6721031, "command_buffers": 2 }, { "compute_ns": 63834, "compute_span_ns": 63834, "calibration_cpu_ns": 521042, "calibration_gpu_ticks": 521042, "compute_passes": 1, "command_buffer_ns": 63874.9916106, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4005083.3486, "command_buffers": 2 }, { "command_buffer_ns": 3969791.69454, "command_buffers": 2 }, { "command_buffer_ns": 3947583.31822, "command_buffers": 2 }, { "command_buffer_ns": 3935333.34136, "command_buffers": 2 }, { "command_buffer_ns": 3955041.69865, "command_buffers": 2 }, { "command_buffer_ns": 3907083.30087, "command_buffers": 2 }, { "command_buffer_ns": 3985291.63795, "command_buffers": 2 }, { "command_buffer_ns": 3976791.63143, "command_buffers": 2 }, { "command_buffer_ns": 3966124.96115, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 66541.7173877, "command_buffers": 2 }, { "command_buffer_ns": 64374.9372102, "command_buffers": 2 }, { "command_buffer_ns": 64291.7002551, "command_buffers": 2 }, { "command_buffer_ns": 63333.3693258, "command_buffers": 2 }, { "command_buffer_ns": 64666.6740067, "command_buffers": 2 }, { "command_buffer_ns": 67083.2814649, "command_buffers": 2 }, { "command_buffer_ns": 63708.2848698, "command_buffers": 2 }, { "command_buffer_ns": 71375.0487193, "command_buffers": 2 }, { "command_buffer_ns": 70708.3381712, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 62.579427321875, 62.0279952271875, 61.6809893471875, 61.48958345875, 61.79752654140625, 61.04817657609375, 62.270181842968746, 62.13736924109375, 61.970702517968746 ], "command_buffer_throughput_us_p50": 61.970702517968746, "command_buffer_throughput_us_p90": 62.332030938749995, "command_buffer_latency_us": [ 66.5417173877, 64.3749372102, 64.2917002551, 63.3333693258, 64.66667400669999, 67.08328146490001, 63.7082848698, 71.3750487193, 70.7083381712 ], "command_buffer_latency_us_p50": 64.66667400669999, "command_buffer_latency_us_p90": 70.84168028082 }, "compute_throughput_us": [ 61.54753125, 61.951828125, 61.369140625, 62.783203125, 61.96809375, 62.414703125, 62.05859375, 61.912765625, 61.99675 ], "compute_throughput_us_p50": 61.96809375, "compute_throughput_us_p90": 62.488403125000005, "command_buffer_throughput_us": [ 61.54948005125, 61.9544271103125, 61.37044329078125, 62.78515593296875, 61.9680986346875, 62.41406299515625, 62.05859426703125, 61.91341071828125, 61.99804738578125 ], "command_buffer_throughput_us_p50": 61.9680986346875, "command_buffer_throughput_us_p90": 62.48828158271875, "compute_latency_us": [ 75.583, 71.334, 63.584, 64.459, 78.458, 70.125, 63.375, 64.416, 63.834 ], "compute_latency_us_p50": 64.459, "compute_latency_us_p90": 76.158, "command_buffer_latency_us": [ 75.5833461881, 71.3749905117, 63.6666663922, 64.5000254735, 78.45833897589999, 70.208334364, 63.4583411738, 64.4166721031, 63.8749916106 ], "command_buffer_latency_us_p50": 64.5000254735, "command_buffer_latency_us_p90": 76.15834474565999, "counter_control_throughput_ratio": 0.9988139530125785, "counter_control_latency_ratio": 1.018420876074009 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 62.158222467, "throughput_us_p90": 62.85229339204, "latency_us_p50": 302.375, "latency_us_p90": 361.75820000000004 }, "slowdown": 0.29237081421733285, "device_compute_slowdown": 0.2975500477953266, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-3ib98y42/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "1ca0ba2a59ee9b200a0f7933e3c823dd873803c26a844f54bebdc043f01bc250", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.890709, "capture_ms": 0.070083, "compile_ms": 53.727041, "allocation_upload_ms": 1.24225, "cold_call_ms": 2.130833, "warmup_ms": 200.157125, "download_ms": 0.309292, "repetitions": 6238, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.69091311318, 4.65025521, 4.27934963129, 4.57462984931, 4.45017778134, 4.00270519397, 4.02319781981, 4.72508560436, 3.98897883937 ], "latency_us": [ 261.209, 220.875, 244.625, 220.125, 215.25, 215.833, 197.375, 208.166, 204.125 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 273417, "compute_span_ns": 273417, "calibration_cpu_ns": 759209, "calibration_gpu_ticks": 759209, "compute_passes": 1, "command_buffer_ns": 273499.987088, "command_buffers": 2 }, { "compute_ns": 262167, "compute_span_ns": 262167, "calibration_cpu_ns": 718875, "calibration_gpu_ticks": 718875, "compute_passes": 1, "command_buffer_ns": 262291.694526, "command_buffers": 2 }, { "compute_ns": 242666, "compute_span_ns": 242666, "calibration_cpu_ns": 672625, "calibration_gpu_ticks": 672625, "compute_passes": 1, "command_buffer_ns": 242708.309088, "command_buffers": 2 }, { "compute_ns": 242958, "compute_span_ns": 242958, "calibration_cpu_ns": 693458, "calibration_gpu_ticks": 693458, "compute_passes": 1, "command_buffer_ns": 242999.987677, "command_buffers": 2 }, { "compute_ns": 261750, "compute_span_ns": 261750, "calibration_cpu_ns": 677875, "calibration_gpu_ticks": 677875, "compute_passes": 1, "command_buffer_ns": 261874.985881, "command_buffers": 2 }, { "compute_ns": 261792, "compute_span_ns": 261792, "calibration_cpu_ns": 669250, "calibration_gpu_ticks": 669250, "compute_passes": 1, "command_buffer_ns": 261916.720774, "command_buffers": 2 }, { "compute_ns": 268917, "compute_span_ns": 268917, "calibration_cpu_ns": 692292, "calibration_gpu_ticks": 692292, "compute_passes": 1, "command_buffer_ns": 269041.629508, "command_buffers": 2 }, { "compute_ns": 295834, "compute_span_ns": 295834, "calibration_cpu_ns": 702750, "calibration_gpu_ticks": 702750, "compute_passes": 1, "command_buffer_ns": 295916.688628, "command_buffers": 2 }, { "compute_ns": 276083, "compute_span_ns": 276083, "calibration_cpu_ns": 670042, "calibration_gpu_ticks": 670042, "compute_passes": 1, "command_buffer_ns": 276124.977972, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6000, "compute_span_ns": 6000, "calibration_cpu_ns": 382375, "calibration_gpu_ticks": 382375, "compute_passes": 1, "command_buffer_ns": 6083.34084973, "command_buffers": 2 }, { "compute_ns": 6458, "compute_span_ns": 6458, "calibration_cpu_ns": 478625, "calibration_gpu_ticks": 478625, "compute_passes": 1, "command_buffer_ns": 6541.72617942, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 378667, "calibration_gpu_ticks": 378667, "compute_passes": 1, "command_buffer_ns": 6583.34465697, "command_buffers": 2 }, { "compute_ns": 6458, "compute_span_ns": 6458, "calibration_cpu_ns": 366500, "calibration_gpu_ticks": 366500, "compute_passes": 1, "command_buffer_ns": 6416.63791612, "command_buffers": 2 }, { "compute_ns": 6459, "compute_span_ns": 6459, "calibration_cpu_ns": 380125, "calibration_gpu_ticks": 380125, "compute_passes": 1, "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 444875, "calibration_gpu_ticks": 444875, "compute_passes": 1, "command_buffer_ns": 6541.6097641, "command_buffers": 2 }, { "compute_ns": 6416, "compute_span_ns": 6416, "calibration_cpu_ns": 369542, "calibration_gpu_ticks": 369542, "compute_passes": 1, "command_buffer_ns": 6375.01943856, "command_buffers": 2 }, { "compute_ns": 6458, "compute_span_ns": 6458, "calibration_cpu_ns": 376542, "calibration_gpu_ticks": 376542, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "compute_ns": 6416, "compute_span_ns": 6416, "calibration_cpu_ns": 396000, "calibration_gpu_ticks": 396000, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 288791.663479, "command_buffers": 2 }, { "command_buffer_ns": 261666.660663, "command_buffers": 2 }, { "command_buffer_ns": 262791.640125, "command_buffers": 2 }, { "command_buffer_ns": 262416.666374, "command_buffers": 2 }, { "command_buffer_ns": 261333.305389, "command_buffers": 2 }, { "command_buffer_ns": 241875.008214, "command_buffers": 2 }, { "command_buffer_ns": 262041.692622, "command_buffers": 2 }, { "command_buffer_ns": 299166.655168, "command_buffers": 2 }, { "command_buffer_ns": 306333.357003, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6291.7242758, "command_buffers": 2 }, { "command_buffer_ns": 6624.96313453, "command_buffers": 2 }, { "command_buffer_ns": 6375.01943856, "command_buffers": 2 }, { "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "command_buffer_ns": 6375.01943856, "command_buffers": 2 }, { "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "command_buffer_ns": 6541.6097641, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.5123697418593745, 4.088541572859375, 4.106119376953124, 4.100260412093751, 4.083332896703125, 3.77929700334375, 4.09440144721875, 4.6744789870000005, 4.786458703171875 ], "command_buffer_throughput_us_p50": 4.100260412093751, "command_buffer_throughput_us_p90": 4.696874930234375, "command_buffer_latency_us": [ 6.2917242758, 6.624963134530001, 6.37501943856, 6.49999128655, 6.49999128655, 6.49999128655, 6.37501943856, 6.54166797176, 6.5416097640999995 ], "command_buffer_latency_us_p50": 6.49999128655, "command_buffer_latency_us_p90": 6.558327004314 }, "compute_throughput_us": [ 4.272140625, 4.096359375, 3.79165625, 3.79621875, 4.08984375, 4.0905, 4.201828125, 4.62240625, 4.313796875 ], "compute_throughput_us_p50": 4.096359375, "compute_throughput_us_p90": 4.37551875, "command_buffer_throughput_us": [ 4.27343729825, 4.09830772696875, 3.7923173295000003, 3.796874807453125, 4.091796654390625, 4.09244876209375, 4.2037754610624996, 4.6236982598125, 4.3144527808125 ], "command_buffer_throughput_us_p50": 4.09830772696875, "command_buffer_throughput_us_p90": 4.3763018766125, "compute_latency_us": [ 6.0, 6.458, 6.5, 6.458, 6.459, 6.5, 6.416, 6.458, 6.416 ], "compute_latency_us_p50": 6.458, "compute_latency_us_p90": 6.5, "command_buffer_latency_us": [ 6.08334084973, 6.54172617942, 6.58334465697, 6.41663791612, 6.49999128655, 6.5416097640999995, 6.37501943856, 6.500049494210001, 6.500049494210001 ], "command_buffer_latency_us_p50": 6.500049494210001, "command_buffer_latency_us_p90": 6.55004987493, "counter_control_throughput_ratio": 0.9891366016771656, "counter_control_latency_ratio": 0.9936467824604763 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.45017778134, "throughput_us_p90": 4.697747611416, "latency_us_p50": 215.833, "latency_us_p90": 247.9418 }, "torch": { "cold_call_ms": 9.033666, "warmup_ms": 200.174083, "repetitions": 893, "throughput_us": [ 30.636851063829784, 29.451754759238522, 29.757978723404257, 30.225596864501682, 38.37826539753639, 33.076987681970884, 31.272910414333705, 31.269036954087348, 30.35288353863382 ], "latency_us": [ 370.584, 324.458, 395.583, 316.33299999999997, 298.75, 303.875, 391.542, 343.708, 477.16700000000003 ], "throughput_us_p50": 30.636851063829784, "throughput_us_p90": 34.137243225083985, "latency_us_p50": 343.708, "latency_us_p90": 411.8998, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1110833.3128504455, "command_buffers": 7 }, { "command_buffer_ns": 1234208.3500698209, "command_buffers": 7 }, { "command_buffer_ns": 1138416.7009964585, "command_buffers": 7 }, { "command_buffer_ns": 1327250.0364109874, "command_buffers": 7 }, { "command_buffer_ns": 1317791.6407585144, "command_buffers": 7 }, { "command_buffer_ns": 1322624.972090125, "command_buffers": 7 }, { "command_buffer_ns": 1485791.6394248605, "command_buffers": 7 }, { "command_buffer_ns": 1325541.699770838, "command_buffers": 7 }, { "command_buffer_ns": 1486416.6732877493, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 89000.03740563989, "command_buffers": 1 }, { "command_buffer_ns": 91291.6730158031, "command_buffers": 1 }, { "command_buffer_ns": 70041.62762314081, "command_buffers": 1 }, { "command_buffer_ns": 85874.98450651765, "command_buffers": 1 }, { "command_buffer_ns": 88499.97539073229, "command_buffers": 1 }, { "command_buffer_ns": 89541.6596904397, "command_buffers": 1 }, { "command_buffer_ns": 88625.00544637442, "command_buffers": 1 }, { "command_buffer_ns": 106666.64456948638, "command_buffers": 1 }, { "command_buffer_ns": 107250.00174716115, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 17.35677051328821, 19.28450546984095, 17.787760953069665, 20.738281818921678, 20.590494386851788, 20.666015188908204, 23.215494366013445, 20.711589058919344, 23.225260520121083 ], "command_buffer_throughput_us_p50": 20.666015188908204, "command_buffer_throughput_us_p90": 23.217447596834973, "command_buffer_latency_us": [ 89.00003740563989, 91.2916730158031, 70.04162762314081, 85.87498450651765, 88.49997539073229, 89.5416596904397, 88.62500544637442, 106.66664456948638, 107.25000174716115 ], "command_buffer_latency_us_p50": 89.00003740563989, "command_buffer_latency_us_p90": 106.78331600502133 }, "throughput": [ { "compute_ns": 3575506.0, "compute_span_ns": 4644584.0, "command_buffer_ns": 4530041.653197259, "calibration_cpu_ns": 5539083.0, "calibration_gpu_ticks": 5539083.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4125667.0, "compute_span_ns": 5294333.0, "command_buffer_ns": 5180458.247195929, "calibration_cpu_ns": 6054083.0, "calibration_gpu_ticks": 6054083.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4280503.0, "compute_span_ns": 5345417.0, "command_buffer_ns": 5262208.345811814, "calibration_cpu_ns": 6025084.0, "calibration_gpu_ticks": 6025084.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4078792.0, "compute_span_ns": 5211875.0, "command_buffer_ns": 5090083.344839513, "calibration_cpu_ns": 5938625.0, "calibration_gpu_ticks": 5938625.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4260039.0, "compute_span_ns": 5446458.0, "command_buffer_ns": 5298041.563946754, "calibration_cpu_ns": 6220792.0, "calibration_gpu_ticks": 6220792.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4617794.0, "compute_span_ns": 5842584.0, "command_buffer_ns": 5727083.480451256, "calibration_cpu_ns": 6606792.0, "calibration_gpu_ticks": 6606792.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4691332.0, "compute_span_ns": 5809333.0, "command_buffer_ns": 5691333.382856101, "calibration_cpu_ns": 6575625.0, "calibration_gpu_ticks": 6575625.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4796500.0, "compute_span_ns": 6064917.0, "command_buffer_ns": 5933791.806455702, "calibration_cpu_ns": 6933875.0, "calibration_gpu_ticks": 6933875.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 5138001.0, "compute_span_ns": 6431958.0, "command_buffer_ns": 6305333.401542157, "calibration_cpu_ns": 7137834.0, "calibration_gpu_ticks": 7137834.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 88542.0, "compute_span_ns": 88542.0, "command_buffer_ns": 88583.32876116037, "calibration_cpu_ns": 567458.0, "calibration_gpu_ticks": 567458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88458.0, "compute_span_ns": 88458.0, "command_buffer_ns": 88416.68022796512, "calibration_cpu_ns": 522208.0, "calibration_gpu_ticks": 522208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70042.0, "compute_span_ns": 70042.0, "command_buffer_ns": 70083.30430835485, "calibration_cpu_ns": 534125.0, "calibration_gpu_ticks": 534125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104291.0, "compute_span_ns": 104291.0, "command_buffer_ns": 104249.97890368104, "calibration_cpu_ns": 781375.0, "calibration_gpu_ticks": 781375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88292.0, "compute_span_ns": 88292.0, "command_buffer_ns": 88333.32685753703, "calibration_cpu_ns": 590250.0, "calibration_gpu_ticks": 590250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88375.0, "compute_span_ns": 88375.0, "command_buffer_ns": 88375.00354275107, "calibration_cpu_ns": 530875.0, "calibration_gpu_ticks": 530875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104542.0, "compute_span_ns": 104542.0, "command_buffer_ns": 104583.33417773247, "calibration_cpu_ns": 547250.0, "calibration_gpu_ticks": 547250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104333.0, "compute_span_ns": 104333.0, "command_buffer_ns": 104333.33227410913, "calibration_cpu_ns": 666292.0, "calibration_gpu_ticks": 666292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104375.0, "compute_span_ns": 104375.0, "command_buffer_ns": 104375.00895932317, "calibration_cpu_ns": 646042.0, "calibration_gpu_ticks": 646042.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 55.86728125, 64.463546875, 66.882859375, 63.731125, 66.563109375, 72.15303125, 73.3020625, 74.9453125, 80.281265625 ], "compute_throughput_us_p50": 66.882859375, "compute_throughput_us_p90": 76.012503125, "command_buffer_throughput_us": [ 70.78190083120717, 80.9446601124364, 82.2220054033096, 79.5325522631174, 82.78189943666803, 89.48567938205088, 88.92708410712657, 92.71549697587034, 98.5208343990962 ], "command_buffer_throughput_us_p50": 82.78189943666803, "command_buffer_throughput_us_p90": 93.87656446051551, "compute_latency_us": [ 88.542, 88.458, 70.042, 104.291, 88.292, 88.375, 104.542, 104.333, 104.375 ], "compute_latency_us_p50": 88.542, "compute_latency_us_p90": 104.4084, "command_buffer_latency_us": [ 88.58332876116037, 88.41668022796512, 70.08330430835485, 104.24997890368104, 88.33332685753703, 88.37500354275107, 104.58333417773247, 104.33333227410913, 104.37500895932317 ], "command_buffer_latency_us_p50": 88.58332876116037, "command_buffer_latency_us_p90": 104.41667400300503, "counter_control_throughput_ratio": 4.197393614216646, "counter_control_latency_ratio": 0.9953178823669449 }, "download_ms": 0.264458, "allocation_upload_ms": 0.699833, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.14525571743872628, "device_compute_slowdown": 0.061246774035668236, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.57425, "warmup_ms": 200.530584, "repetitions": 905, "throughput_us": [ 29.71358232044199, 29.238903867403312, 29.154097237569058, 28.827807734806626, 29.494428729281765, 29.107643093922654, 29.481077348066297, 29.052624309392264, 29.096040883977903 ], "latency_us": [ 313.042, 307.08299999999997, 344.875, 247.416, 325.042, 324.791, 322.958, 313.416, 344.625 ], "throughput_us_p50": 29.154097237569058, "throughput_us_p90": 29.53825944751381, "latency_us_p50": 322.958, "latency_us_p90": 344.675, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1050625.1128390431, "command_buffers": 7 }, { "command_buffer_ns": 1048416.6559763253, "command_buffers": 7 }, { "command_buffer_ns": 1054083.346389234, "command_buffers": 7 }, { "command_buffer_ns": 1047500.00173226, "command_buffers": 7 }, { "command_buffer_ns": 1072833.1981226802, "command_buffers": 7 }, { "command_buffer_ns": 1045583.3398737013, "command_buffers": 7 }, { "command_buffer_ns": 1147749.9501779675, "command_buffers": 7 }, { "command_buffer_ns": 1144708.3088569343, "command_buffers": 7 }, { "command_buffer_ns": 1052666.688337922, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 60499.99501556158, "command_buffers": 1 }, { "command_buffer_ns": 62875.04220381379, "command_buffers": 1 }, { "command_buffer_ns": 65375.0030323863, "command_buffers": 1 }, { "command_buffer_ns": 74499.98520314693, "command_buffers": 1 }, { "command_buffer_ns": 61166.64735600352, "command_buffers": 1 }, { "command_buffer_ns": 80208.29409360886, "command_buffers": 1 }, { "command_buffer_ns": 77124.97608736157, "command_buffers": 1 }, { "command_buffer_ns": 70124.9809935689, "command_buffers": 1 }, { "command_buffer_ns": 77000.00423938036, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.41601738811005, 16.381510249630082, 16.470052287331782, 16.367187527066562, 16.763018720666878, 16.337239685526583, 17.933592971530743, 17.8860673258896, 16.447917005280033 ], "command_buffer_throughput_us_p50": 16.447917005280033, "command_buffer_throughput_us_p90": 17.895572455017827, "command_buffer_latency_us": [ 60.49999501556158, 62.87504220381379, 65.3750030323863, 74.49998520314693, 61.16664735600352, 80.20829409360886, 77.12497608736157, 70.1249809935689, 77.00000423938036 ], "command_buffer_latency_us_p50": 70.1249809935689, "command_buffer_latency_us_p90": 77.74163968861103 }, "throughput": [ { "compute_ns": 3824667.0, "compute_span_ns": 4925083.0, "command_buffer_ns": 4803708.288818598, "calibration_cpu_ns": 5588208.0, "calibration_gpu_ticks": 5588208.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3583957.0, "compute_span_ns": 4679333.0, "command_buffer_ns": 4558875.109069049, "calibration_cpu_ns": 5359625.0, "calibration_gpu_ticks": 5359625.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3685380.0, "compute_span_ns": 4870000.0, "command_buffer_ns": 4755958.332680166, "calibration_cpu_ns": 5625125.0, "calibration_gpu_ticks": 5625125.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3801419.0, "compute_span_ns": 4899417.0, "command_buffer_ns": 4797791.596502066, "calibration_cpu_ns": 5580875.0, "calibration_gpu_ticks": 5580875.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3736039.0, "compute_span_ns": 4919000.0, "command_buffer_ns": 4796499.968506396, "calibration_cpu_ns": 5573166.0, "calibration_gpu_ticks": 5573166.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3883582.0, "compute_span_ns": 5033875.0, "command_buffer_ns": 4916249.949019402, "calibration_cpu_ns": 5662125.0, "calibration_gpu_ticks": 5662125.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3622456.0, "compute_span_ns": 4646000.0, "command_buffer_ns": 4541375.092230737, "calibration_cpu_ns": 5249875.0, "calibration_gpu_ticks": 5249875.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3976536.0, "compute_span_ns": 5219291.0, "command_buffer_ns": 5095666.740089655, "calibration_cpu_ns": 5960500.0, "calibration_gpu_ticks": 5960500.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4215167.0, "compute_span_ns": 5354500.0, "command_buffer_ns": 5252291.681244969, "calibration_cpu_ns": 6021166.0, "calibration_gpu_ticks": 6021166.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 54000.0, "compute_span_ns": 54000.0, "command_buffer_ns": 54000.00372901559, "calibration_cpu_ns": 524083.0, "calibration_gpu_ticks": 524083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 58125.0, "compute_span_ns": 58125.0, "command_buffer_ns": 58125.00603497028, "calibration_cpu_ns": 515917.0, "calibration_gpu_ticks": 515917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 73625.0, "compute_span_ns": 73625.0, "command_buffer_ns": 73625.00764429569, "calibration_cpu_ns": 698333.0, "calibration_gpu_ticks": 698333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 77917.0, "compute_span_ns": 77917.0, "command_buffer_ns": 77916.65848344564, "calibration_cpu_ns": 581875.0, "calibration_gpu_ticks": 581875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 58084.0, "compute_span_ns": 58084.0, "command_buffer_ns": 58125.00603497028, "calibration_cpu_ns": 512333.0, "calibration_gpu_ticks": 512333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56709.0, "compute_span_ns": 56709.0, "command_buffer_ns": 56749.96646121144, "calibration_cpu_ns": 492833.0, "calibration_gpu_ticks": 492833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71459.0, "compute_span_ns": 71459.0, "command_buffer_ns": 71499.96235966682, "calibration_cpu_ns": 517333.0, "calibration_gpu_ticks": 517333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 76750.0, "compute_span_ns": 76750.0, "command_buffer_ns": 76750.00233575702, "calibration_cpu_ns": 513042.0, "calibration_gpu_ticks": 513042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 76708.0, "compute_span_ns": 76708.0, "command_buffer_ns": 76666.64896532893, "calibration_cpu_ns": 523458.0, "calibration_gpu_ticks": 523458.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 59.760421875, 55.999328125, 57.5840625, 59.397171875, 58.375609375, 60.68096875, 56.600875, 62.133375, 65.861984375 ], "compute_throughput_us_p50": 59.397171875, "compute_throughput_us_p90": 62.879096875, "command_buffer_throughput_us": [ 75.05794201279059, 71.2324235792039, 74.31184894812759, 74.96549369534478, 74.94531200791243, 76.81640545342816, 70.95898581610527, 79.61979281390086, 82.06705751945265 ], "command_buffer_throughput_us_p50": 74.96549369534478, "command_buffer_throughput_us_p90": 80.10924575501122, "compute_latency_us": [ 54.0, 58.125, 73.625, 77.917, 58.084, 56.709, 71.459, 76.75, 76.708 ], "compute_latency_us_p50": 71.459, "compute_latency_us_p90": 76.9834, "command_buffer_latency_us": [ 54.00000372901559, 58.12500603497028, 73.62500764429569, 77.91665848344564, 58.12500603497028, 56.74996646121144, 71.49996235966682, 76.75000233575702, 76.66664896532893 ], "command_buffer_latency_us_p50": 71.49996235966682, "command_buffer_latency_us_p90": 76.98333356529474, "counter_control_throughput_ratio": 4.511937646080566, "counter_control_latency_ratio": 0.9502728782349275 }, "download_ms": 0.2885, "allocation_upload_ms": 0.804458, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-m2o1qkif/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "d45e9e72129e03853b1c708b1bdcbc59be9547b0d0e02ca6bdc2eb0b41c08926", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.812166, "capture_ms": 0.070209, "compile_ms": 56.524292, "allocation_upload_ms": 1.227167, "cold_call_ms": 1.812917, "warmup_ms": 200.144542, "download_ms": 0.290958, "repetitions": 6146, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.88454555809, 4.50686088513, 4.88386755613, 4.89137910836, 4.99125447445, 4.68943898471, 4.37012561015, 4.37208493329, 4.34260087862 ], "latency_us": [ 250.791, 311.542, 206.334, 212.25, 230.875, 206.75, 207.542, 207.292, 205.084 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 268875, "compute_span_ns": 268875, "calibration_cpu_ns": 721875, "calibration_gpu_ticks": 721875, "compute_passes": 1, "command_buffer_ns": 268958.392553, "command_buffers": 2 }, { "compute_ns": 268333, "compute_span_ns": 268333, "calibration_cpu_ns": 698917, "calibration_gpu_ticks": 698917, "compute_passes": 1, "command_buffer_ns": 268375.035375, "command_buffers": 2 }, { "compute_ns": 291292, "compute_span_ns": 291292, "calibration_cpu_ns": 790334, "calibration_gpu_ticks": 790334, "compute_passes": 1, "command_buffer_ns": 291458.331048, "command_buffers": 2 }, { "compute_ns": 291917, "compute_span_ns": 291917, "calibration_cpu_ns": 751292, "calibration_gpu_ticks": 751292, "compute_passes": 1, "command_buffer_ns": 292083.306704, "command_buffers": 2 }, { "compute_ns": 294417, "compute_span_ns": 294417, "calibration_cpu_ns": 766542, "calibration_gpu_ticks": 766542, "compute_passes": 1, "command_buffer_ns": 294500.030577, "command_buffers": 2 }, { "compute_ns": 291250, "compute_span_ns": 291250, "calibration_cpu_ns": 703292, "calibration_gpu_ticks": 703292, "compute_passes": 1, "command_buffer_ns": 291291.682515, "command_buffers": 2 }, { "compute_ns": 266833, "compute_span_ns": 266833, "calibration_cpu_ns": 759500, "calibration_gpu_ticks": 759500, "compute_passes": 1, "command_buffer_ns": 266874.965746, "command_buffers": 2 }, { "compute_ns": 266583, "compute_span_ns": 266583, "calibration_cpu_ns": 775250, "calibration_gpu_ticks": 775250, "compute_passes": 1, "command_buffer_ns": 266708.317213, "command_buffers": 2 }, { "compute_ns": 312875, "compute_span_ns": 312875, "calibration_cpu_ns": 726916, "calibration_gpu_ticks": 726916, "compute_passes": 1, "command_buffer_ns": 312999.938615, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 382250, "calibration_gpu_ticks": 382250, "compute_passes": 1, "command_buffer_ns": 6749.93498251, "command_buffers": 2 }, { "compute_ns": 7083, "compute_span_ns": 7083, "calibration_cpu_ns": 467208, "calibration_gpu_ticks": 467208, "compute_passes": 1, "command_buffer_ns": 7166.64362699, "command_buffers": 2 }, { "compute_ns": 7041, "compute_span_ns": 7041, "calibration_cpu_ns": 372750, "calibration_gpu_ticks": 372750, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 381125, "calibration_gpu_ticks": 381125, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 10042, "compute_span_ns": 10042, "calibration_cpu_ns": 380667, "calibration_gpu_ticks": 380667, "compute_passes": 1, "command_buffer_ns": 10083.3713077, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 373458, "calibration_gpu_ticks": 373458, "compute_passes": 1, "command_buffer_ns": 6958.26020092, "command_buffers": 2 }, { "compute_ns": 7083, "compute_span_ns": 7083, "calibration_cpu_ns": 399542, "calibration_gpu_ticks": 399542, "compute_passes": 1, "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "compute_ns": 7500, "compute_span_ns": 7500, "calibration_cpu_ns": 457042, "calibration_gpu_ticks": 457042, "compute_passes": 1, "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 421375, "calibration_gpu_ticks": 421375, "compute_passes": 1, "command_buffer_ns": 7083.34846422, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 295750.040095, "command_buffers": 2 }, { "command_buffer_ns": 269000.011031, "command_buffers": 2 }, { "command_buffer_ns": 291791.686323, "command_buffers": 2 }, { "command_buffer_ns": 292208.394967, "command_buffers": 2 }, { "command_buffer_ns": 292291.69013, "command_buffers": 2 }, { "command_buffer_ns": 293499.964755, "command_buffers": 2 }, { "command_buffer_ns": 296291.66238, "command_buffers": 2 }, { "command_buffer_ns": 292124.983389, "command_buffers": 2 }, { "command_buffer_ns": 292125.041597, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6874.96503815, "command_buffers": 2 }, { "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "command_buffer_ns": 7624.97074902, "command_buffers": 2 }, { "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "command_buffer_ns": 7166.64362699, "command_buffers": 2 }, { "command_buffer_ns": 7333.29216018, "command_buffers": 2 }, { "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "command_buffer_ns": 7083.34846422, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.6210943764843755, 4.203125172359375, 4.559245098796875, 4.565756171359375, 4.56705765828125, 4.585936949296875, 4.6295572246875, 4.564452865453125, 4.564453774953124 ], "command_buffer_throughput_us_p50": 4.565756171359375, "command_buffer_throughput_us_p90": 4.622786946125, "command_buffer_latency_us": [ 6.87496503815, 6.99999509379, 7.62497074902, 7.12502514943, 7.1667018346499995, 7.16664362699, 7.33329216018, 7.208320312200001, 7.08334846422 ], "command_buffer_latency_us_p50": 7.16664362699, "command_buffer_latency_us_p90": 7.391627877948 }, "compute_throughput_us": [ 4.201171875, 4.192703125, 4.5514375, 4.561203125, 4.600265625, 4.55078125, 4.169265625, 4.165359375, 4.888671875 ], "compute_throughput_us_p50": 4.55078125, "compute_throughput_us_p90": 4.6579468749999995, "command_buffer_throughput_us": [ 4.202474883640625, 4.193359927734375, 4.554036422625001, 4.56380166725, 4.601562977765625, 4.551432539296875, 4.16992133978125, 4.167317456453125, 4.890624040859375 ], "command_buffer_throughput_us_p50": 4.551432539296875, "command_buffer_throughput_us_p90": 4.659375190384376, "compute_latency_us": [ 6.708, 7.083, 7.041, 7.125, 10.042, 6.958, 7.083, 7.5, 7.042 ], "compute_latency_us_p50": 7.083, "compute_latency_us_p90": 8.0084, "command_buffer_latency_us": [ 6.74993498251, 7.16664362699, 6.99999509379, 7.208320312200001, 10.083371307699998, 6.95826020092, 7.12502514943, 7.54167558625, 7.08334846422 ], "command_buffer_latency_us_p50": 7.12502514943, "command_buffer_latency_us_p90": 8.050014730540001, "counter_control_throughput_ratio": 0.9976766705190655, "counter_control_latency_ratio": 1.0 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.68943898471, "throughput_us_p90": 4.911354181578, "latency_us_p50": 207.542, "latency_us_p90": 262.9412 }, "slowdown": 0.1608500838320253, "device_compute_slowdown": 0.07661612676739923, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-n2ljwpkp/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7bac43b2781c01105a8e678fa6161555c0e89cbd2b0a98679fdc42bb1bd74b45", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 25.622375, "capture_ms": 0.07975, "compile_ms": 48.585083, "allocation_upload_ms": 1.662042, "cold_call_ms": 5.955792, "warmup_ms": 200.561125, "download_ms": 1.015708, "repetitions": 1995, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 15.2578320802, 14.7851924812, 15.2214285714, 14.7811192982, 14.3850250627, 14.3116330827, 14.5105057644, 15.1328947368, 14.9272972431 ], "latency_us": [ 272.208, 232.75, 284.833, 233.666, 228, 235.833, 254.625, 244.333, 233.125 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 946542, "compute_span_ns": 946542, "calibration_cpu_ns": 1387666, "calibration_gpu_ticks": 1387666, "compute_passes": 1, "command_buffer_ns": 946625.019424, "command_buffers": 2 }, { "compute_ns": 958416, "compute_span_ns": 958416, "calibration_cpu_ns": 1396958, "calibration_gpu_ticks": 1396958, "compute_passes": 1, "command_buffer_ns": 958416.669164, "command_buffers": 2 }, { "compute_ns": 928791, "compute_span_ns": 928791, "calibration_cpu_ns": 1499208, "calibration_gpu_ticks": 1499208, "compute_passes": 1, "command_buffer_ns": 928875.058889, "command_buffers": 2 }, { "compute_ns": 935125, "compute_span_ns": 935125, "calibration_cpu_ns": 1386459, "calibration_gpu_ticks": 1386459, "compute_passes": 1, "command_buffer_ns": 935249.961913, "command_buffers": 2 }, { "compute_ns": 953375, "compute_span_ns": 953375, "calibration_cpu_ns": 1396333, "calibration_gpu_ticks": 1396333, "compute_passes": 1, "command_buffer_ns": 953375.012614, "command_buffers": 2 }, { "compute_ns": 959959, "compute_span_ns": 959959, "calibration_cpu_ns": 1409208, "calibration_gpu_ticks": 1409208, "compute_passes": 1, "command_buffer_ns": 960041.652434, "command_buffers": 2 }, { "compute_ns": 972375, "compute_span_ns": 972375, "calibration_cpu_ns": 1395875, "calibration_gpu_ticks": 1395875, "compute_passes": 1, "command_buffer_ns": 972374.982666, "command_buffers": 2 }, { "compute_ns": 975958, "compute_span_ns": 975958, "calibration_cpu_ns": 1360292, "calibration_gpu_ticks": 1360292, "compute_passes": 1, "command_buffer_ns": 975958.362687, "command_buffers": 2 }, { "compute_ns": 960584, "compute_span_ns": 960584, "calibration_cpu_ns": 1391416, "calibration_gpu_ticks": 1391416, "compute_passes": 1, "command_buffer_ns": 960708.362982, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18667, "compute_span_ns": 18667, "calibration_cpu_ns": 433042, "calibration_gpu_ticks": 433042, "compute_passes": 1, "command_buffer_ns": 18749.9681488, "command_buffers": 2 }, { "compute_ns": 18667, "compute_span_ns": 18667, "calibration_cpu_ns": 386333, "calibration_gpu_ticks": 386333, "compute_passes": 1, "command_buffer_ns": 18708.3496712, "command_buffers": 2 }, { "compute_ns": 18666, "compute_span_ns": 18666, "calibration_cpu_ns": 434625, "calibration_gpu_ticks": 434625, "compute_passes": 1, "command_buffer_ns": 18708.3496712, "command_buffers": 2 }, { "compute_ns": 18541, "compute_span_ns": 18541, "calibration_cpu_ns": 411417, "calibration_gpu_ticks": 411417, "compute_passes": 1, "command_buffer_ns": 18583.3778232, "command_buffers": 2 }, { "compute_ns": 19042, "compute_span_ns": 19042, "calibration_cpu_ns": 454000, "calibration_gpu_ticks": 454000, "compute_passes": 1, "command_buffer_ns": 19124.9419004, "command_buffers": 2 }, { "compute_ns": 18709, "compute_span_ns": 18709, "calibration_cpu_ns": 445125, "calibration_gpu_ticks": 445125, "compute_passes": 1, "command_buffer_ns": 18874.9399967, "command_buffers": 2 }, { "compute_ns": 18833, "compute_span_ns": 18833, "calibration_cpu_ns": 421167, "calibration_gpu_ticks": 421167, "compute_passes": 1, "command_buffer_ns": 18916.6748896, "command_buffers": 2 }, { "compute_ns": 18583, "compute_span_ns": 18583, "calibration_cpu_ns": 416625, "calibration_gpu_ticks": 416625, "compute_passes": 1, "command_buffer_ns": 18666.672986, "command_buffers": 2 }, { "compute_ns": 18584, "compute_span_ns": 18584, "calibration_cpu_ns": 457333, "calibration_gpu_ticks": 457333, "compute_passes": 1, "command_buffer_ns": 18624.9963008, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1033166.71448, "command_buffers": 2 }, { "command_buffer_ns": 924791.675061, "command_buffers": 2 }, { "command_buffer_ns": 1003833.31301, "command_buffers": 2 }, { "command_buffer_ns": 943124.992773, "command_buffers": 2 }, { "command_buffer_ns": 1004583.31872, "command_buffers": 2 }, { "command_buffer_ns": 944416.620769, "command_buffers": 2 }, { "command_buffer_ns": 1027874.99782, "command_buffers": 2 }, { "command_buffer_ns": 940875.033848, "command_buffers": 2 }, { "command_buffer_ns": 1059708.2437, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 16833.3062902, "command_buffers": 2 }, { "command_buffer_ns": 18749.9681488, "command_buffers": 2 }, { "command_buffer_ns": 18666.672986, "command_buffers": 2 }, { "command_buffer_ns": 18999.9700524, "command_buffers": 2 }, { "command_buffer_ns": 18666.672986, "command_buffers": 2 }, { "command_buffer_ns": 18708.2914636, "command_buffers": 2 }, { "command_buffer_ns": 21583.3424591, "command_buffers": 2 }, { "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "command_buffer_ns": 18500.0244528, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 16.143229913749998, 14.449869922828126, 15.684895515781252, 14.736328012078125, 15.696614355, 14.756509699515625, 16.0605468409375, 14.701172403875, 16.5579413078125 ], "command_buffer_throughput_us_p50": 15.684895515781252, "command_buffer_throughput_us_p90": 16.226172192562498, "command_buffer_latency_us": [ 16.8333062902, 18.749968148799997, 18.666672986000002, 18.9999700524, 18.666672986000002, 18.7082914636, 21.5833424591, 18.624996300800003, 18.5000244528 ], "command_buffer_latency_us_p50": 18.666672986000002, "command_buffer_latency_us_p90": 19.51664453374 }, "compute_throughput_us": [ 14.78971875, 14.97525, 14.512359375, 14.611328125, 14.896484375, 14.999359375, 15.193359375, 15.24934375, 15.009125 ], "compute_throughput_us_p50": 14.97525, "compute_throughput_us_p90": 15.20455625, "command_buffer_throughput_us": [ 14.7910159285, 14.975260455687499, 14.513672795140625, 14.613280654890625, 14.89648457209375, 15.000650819281251, 15.19335910415625, 15.249349416984375, 15.01106817159375 ], "command_buffer_throughput_us_p50": 14.975260455687499, "command_buffer_throughput_us_p90": 15.204557166721875, "compute_latency_us": [ 18.667, 18.667, 18.666, 18.541, 19.042, 18.709, 18.833, 18.583, 18.584 ], "compute_latency_us_p50": 18.667, "compute_latency_us_p90": 18.8748, "command_buffer_latency_us": [ 18.749968148799997, 18.7083496712, 18.7083496712, 18.5833778232, 19.1249419004, 18.8749399967, 18.9166748896, 18.666672986000002, 18.624996300800003 ], "command_buffer_latency_us_p50": 18.7083496712, "command_buffer_latency_us_p90": 18.95832829176, "counter_control_throughput_ratio": 0.9490253270667011, "counter_control_latency_ratio": 1.0022376748175896 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 14.7851924812, "throughput_us_p90": 15.22870927316, "latency_us_p50": 235.833, "latency_us_p90": 274.733 }, "torch": { "cold_call_ms": 6.748459, "warmup_ms": 200.821667, "repetitions": 884, "throughput_us": [ 33.64691742081448, 33.46460294117647, 33.36420588235294, 33.525263574660634, 35.169354072398185, 33.73274886877828, 33.779742081447964, 33.57866628959276, 33.62697963800905 ], "latency_us": [ 356.66700000000003, 333.625, 319.125, 543.584, 353.292, 331.25, 405.875, 327.084, 341.0 ], "throughput_us_p50": 33.62697963800905, "throughput_us_p90": 34.05766447963801, "latency_us_p50": 341.0, "latency_us_p90": 433.4168, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1861874.8290464282, "command_buffers": 11 }, { "command_buffer_ns": 1886708.3126679063, "command_buffers": 11 }, { "command_buffer_ns": 1839458.1857137382, "command_buffers": 11 }, { "command_buffer_ns": 1837375.049944967, "command_buffers": 11 }, { "command_buffer_ns": 1841083.4018141031, "command_buffers": 11 }, { "command_buffer_ns": 1838083.437178284, "command_buffers": 11 }, { "command_buffer_ns": 1858250.0633783638, "command_buffers": 11 }, { "command_buffer_ns": 1870500.0402405858, "command_buffers": 11 }, { "command_buffer_ns": 1838750.1477263868, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 48249.95994567871, "command_buffers": 1 }, { "command_buffer_ns": 49291.644245386124, "command_buffers": 1 }, { "command_buffer_ns": 49291.644245386124, "command_buffers": 1 }, { "command_buffer_ns": 49999.97327104211, "command_buffers": 1 }, { "command_buffer_ns": 49541.646149009466, "command_buffers": 1 }, { "command_buffer_ns": 49333.320930600166, "command_buffers": 1 }, { "command_buffer_ns": 49499.96946379542, "command_buffers": 1 }, { "command_buffer_ns": 48541.69674217701, "command_buffers": 1 }, { "command_buffer_ns": 49374.99761581421, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.09179420385044, 29.479817385436036, 28.74153415177716, 28.70898515539011, 28.76692815334536, 28.720053705910686, 29.035157240286935, 29.226563128759153, 28.730471058224794 ], "command_buffer_throughput_us_p50": 28.76692815334536, "command_buffer_throughput_us_p90": 29.27721398009453, "command_buffer_latency_us": [ 48.24995994567871, 49.291644245386124, 49.291644245386124, 49.99997327104211, 49.541646149009466, 49.333320930600166, 49.49996946379542, 48.54169674217701, 49.37499761581421 ], "command_buffer_latency_us_p50": 49.333320930600166, "command_buffer_latency_us_p90": 49.633311573415995 }, "throughput": [ { "compute_ns": 2967579.0, "compute_span_ns": 3864417.0, "command_buffer_ns": 3713333.3389647305, "calibration_cpu_ns": 4804500.0, "calibration_gpu_ticks": 4804500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2983498.0, "compute_span_ns": 3867917.0, "command_buffer_ns": 3736083.22115615, "calibration_cpu_ns": 4660292.0, "calibration_gpu_ticks": 4660292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2981835.0, "compute_span_ns": 3934333.0, "command_buffer_ns": 3783458.2617506385, "calibration_cpu_ns": 4712375.0, "calibration_gpu_ticks": 4712375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2971666.0, "compute_span_ns": 3875916.0, "command_buffer_ns": 3733958.350494504, "calibration_cpu_ns": 4718417.0, "calibration_gpu_ticks": 4718417.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2985539.0, "compute_span_ns": 3915792.0, "command_buffer_ns": 3730249.940417707, "calibration_cpu_ns": 4592833.0, "calibration_gpu_ticks": 4592833.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2954252.0, "compute_span_ns": 3829750.0, "command_buffer_ns": 3690125.0132359564, "calibration_cpu_ns": 4617333.0, "calibration_gpu_ticks": 4617333.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3007250.0, "compute_span_ns": 3816500.0, "command_buffer_ns": 3694291.6340194643, "calibration_cpu_ns": 4472875.0, "calibration_gpu_ticks": 4472875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3015038.0, "compute_span_ns": 3930625.0, "command_buffer_ns": 3772374.999243766, "calibration_cpu_ns": 4686792.0, "calibration_gpu_ticks": 4686792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3005666.0, "compute_span_ns": 3874166.0, "command_buffer_ns": 3724458.5109874606, "calibration_cpu_ns": 4489458.0, "calibration_gpu_ticks": 4489458.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 48417.0, "compute_span_ns": 48417.0, "command_buffer_ns": 48458.343371748924, "calibration_cpu_ns": 569584.0, "calibration_gpu_ticks": 569584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47416.0, "compute_span_ns": 47416.0, "command_buffer_ns": 47374.98238682747, "calibration_cpu_ns": 482625.0, "calibration_gpu_ticks": 482625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49500.0, "compute_span_ns": 49500.0, "command_buffer_ns": 49500.02767145634, "calibration_cpu_ns": 480166.0, "calibration_gpu_ticks": 480166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49666.0, "compute_span_ns": 49666.0, "command_buffer_ns": 49624.99951943755, "calibration_cpu_ns": 488083.0, "calibration_gpu_ticks": 488083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47792.0, "compute_span_ns": 47792.0, "command_buffer_ns": 47791.63282364607, "calibration_cpu_ns": 457542.0, "calibration_gpu_ticks": 457542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 48042.0, "compute_span_ns": 48042.0, "command_buffer_ns": 48083.311412483454, "calibration_cpu_ns": 476208.0, "calibration_gpu_ticks": 476208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50125.0, "compute_span_ns": 50125.0, "command_buffer_ns": 50125.00332668424, "calibration_cpu_ns": 520000.0, "calibration_gpu_ticks": 520000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50333.0, "compute_span_ns": 50333.0, "command_buffer_ns": 50291.651859879494, "calibration_cpu_ns": 472000.0, "calibration_gpu_ticks": 472000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47917.0, "compute_span_ns": 47917.0, "command_buffer_ns": 47958.281356841326, "calibration_cpu_ns": 509583.0, "calibration_gpu_ticks": 509583.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 46.368421875, 46.61715625, 46.591171875, 46.43228125, 46.649046875, 46.1601875, 46.98828125, 47.10996875, 46.96353125 ], "compute_throughput_us_p50": 46.61715625, "compute_throughput_us_p90": 47.01261875, "command_buffer_throughput_us": [ 58.020833421323914, 58.376300330564845, 59.116535339853726, 58.343099226476625, 58.28515531902667, 57.65820333181182, 57.72330678155413, 58.943359363183845, 58.19466423417907 ], "command_buffer_throughput_us_p50": 58.28515531902667, "command_buffer_throughput_us_p90": 58.97799455851782, "compute_latency_us": [ 48.417, 47.416, 49.5, 49.666, 47.792, 48.042, 50.125, 50.333, 47.917 ], "compute_latency_us_p50": 48.417, "compute_latency_us_p90": 50.1666, "command_buffer_latency_us": [ 48.458343371748924, 47.37498238682747, 49.50002767145634, 49.62499951943755, 47.79163282364607, 48.083311412483454, 50.12500332668424, 50.291651859879494, 47.958281356841326 ], "command_buffer_latency_us_p50": 48.458343371748924, "command_buffer_latency_us_p90": 50.15833303332329, "counter_control_throughput_ratio": 2.0167735461575074, "counter_control_latency_ratio": 0.9925005209588437 }, "download_ms": 0.600625, "allocation_upload_ms": 1.101291, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.43968244071757456, "device_compute_slowdown": 0.3212390288178507, "valid": true, "round": 2, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.438167, "warmup_ms": 200.422875, "repetitions": 713, "throughput_us": [ 45.885227208976154, 33.455995792426364, 33.56358064516129, 33.28342636746143, 33.53266760168302, 33.272206171107996, 33.33058625525946, 33.27062973352034, 33.58216549789621 ], "latency_us": [ 333.5, 305.291, 299.417, 290.66700000000003, 285.625, 306.291, 295.833, 287.292, 281.583 ], "throughput_us_p50": 33.455995792426364, "throughput_us_p90": 36.0427778401122, "latency_us_p50": 295.833, "latency_us_p90": 311.7328, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1866458.2748897374, "command_buffers": 11 }, { "command_buffer_ns": 1864583.2315087318, "command_buffers": 11 }, { "command_buffer_ns": 1836708.3393968642, "command_buffers": 11 }, { "command_buffer_ns": 1851416.7750254273, "command_buffers": 11 }, { "command_buffer_ns": 1867791.695985943, "command_buffers": 11 }, { "command_buffer_ns": 1851833.2508392632, "command_buffers": 11 }, { "command_buffer_ns": 1846791.5360815823, "command_buffers": 11 }, { "command_buffer_ns": 1827000.0582560897, "command_buffers": 11 }, { "command_buffer_ns": 1841833.2911096513, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 49041.700549423695, "command_buffers": 1 }, { "command_buffer_ns": 49875.001423060894, "command_buffers": 1 }, { "command_buffer_ns": 53083.29127728939, "command_buffers": 1 }, { "command_buffer_ns": 50291.71006754041, "command_buffers": 1 }, { "command_buffer_ns": 47708.3376608789, "command_buffers": 1 }, { "command_buffer_ns": 49624.99951943755, "command_buffers": 1 }, { "command_buffer_ns": 49333.320930600166, "command_buffers": 1 }, { "command_buffer_ns": 48666.668590158224, "command_buffers": 1 }, { "command_buffer_ns": 47958.281356841326, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.163410545152146, 29.134112992323935, 28.698567803076003, 28.928387109772302, 29.18424524978036, 28.934894544363488, 28.856117751274724, 28.5468759102514, 28.778645173588302 ], "command_buffer_throughput_us_p50": 28.928387109772302, "command_buffer_throughput_us_p90": 29.16757748607779, "command_buffer_latency_us": [ 49.041700549423695, 49.875001423060894, 53.08329127728939, 50.29171006754041, 47.7083376608789, 49.62499951943755, 49.333320930600166, 48.666668590158224, 47.958281356841326 ], "command_buffer_latency_us_p50": 49.333320930600166, "command_buffer_latency_us_p90": 50.850026309490204 }, "throughput": [ { "compute_ns": 3020500.0, "compute_span_ns": 4004375.0, "command_buffer_ns": 3838374.9779313803, "calibration_cpu_ns": 4633125.0, "calibration_gpu_ticks": 4633125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3021414.0, "compute_span_ns": 3978250.0, "command_buffer_ns": 3811250.0915303826, "calibration_cpu_ns": 4572041.0, "calibration_gpu_ticks": 4572041.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2993667.0, "compute_span_ns": 3853250.0, "command_buffer_ns": 3714583.464898169, "calibration_cpu_ns": 4468666.0, "calibration_gpu_ticks": 4468666.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3031956.0, "compute_span_ns": 3976875.0, "command_buffer_ns": 3826916.6834652424, "calibration_cpu_ns": 4589666.0, "calibration_gpu_ticks": 4589666.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3031748.0, "compute_span_ns": 3937875.0, "command_buffer_ns": 3797249.9849274755, "calibration_cpu_ns": 4572375.0, "calibration_gpu_ticks": 4572375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3013544.0, "compute_span_ns": 4005667.0, "command_buffer_ns": 3855916.7296625674, "calibration_cpu_ns": 4639083.0, "calibration_gpu_ticks": 4639083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3018250.0, "compute_span_ns": 3901500.0, "command_buffer_ns": 3760458.3194479346, "calibration_cpu_ns": 4502542.0, "calibration_gpu_ticks": 4502542.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3073831.0, "compute_span_ns": 4049792.0, "command_buffer_ns": 3876624.919939786, "calibration_cpu_ns": 4852708.0, "calibration_gpu_ticks": 4852708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3025377.0, "compute_span_ns": 3977667.0, "command_buffer_ns": 3790041.6646152735, "calibration_cpu_ns": 4591416.0, "calibration_gpu_ticks": 4591416.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 46834.0, "compute_span_ns": 46834.0, "command_buffer_ns": 46875.0367872417, "calibration_cpu_ns": 505291.0, "calibration_gpu_ticks": 505291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49916.0, "compute_span_ns": 49916.0, "command_buffer_ns": 49875.001423060894, "calibration_cpu_ns": 496417.0, "calibration_gpu_ticks": 496417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49666.0, "compute_span_ns": 49666.0, "command_buffer_ns": 49624.99951943755, "calibration_cpu_ns": 499834.0, "calibration_gpu_ticks": 499834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50666.0, "compute_span_ns": 50666.0, "command_buffer_ns": 50625.00713393092, "calibration_cpu_ns": 502791.0, "calibration_gpu_ticks": 502791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49708.0, "compute_span_ns": 49708.0, "command_buffer_ns": 49666.676204651594, "calibration_cpu_ns": 494750.0, "calibration_gpu_ticks": 494750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49750.0, "compute_span_ns": 49750.0, "command_buffer_ns": 49749.971367418766, "calibration_cpu_ns": 497791.0, "calibration_gpu_ticks": 497791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50542.0, "compute_span_ns": 50542.0, "command_buffer_ns": 50541.653763502836, "calibration_cpu_ns": 565334.0, "calibration_gpu_ticks": 565334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47916.0, "compute_span_ns": 47916.0, "command_buffer_ns": 47874.986194074154, "calibration_cpu_ns": 494208.0, "calibration_gpu_ticks": 494208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49750.0, "compute_span_ns": 49750.0, "command_buffer_ns": 49749.971367418766, "calibration_cpu_ns": 506291.0, "calibration_gpu_ticks": 506291.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.1953125, 47.20959375, 46.776046875, 47.3743125, 47.3710625, 47.086625, 47.16015625, 48.028609375, 47.271515625 ], "compute_throughput_us_p50": 47.20959375, "compute_throughput_us_p90": 47.505171875, "command_buffer_throughput_us": [ 59.97460903017782, 59.55078268016223, 58.04036663903389, 59.79557317914441, 59.332031014491804, 60.248698900977615, 58.75716124137398, 60.57226437405916, 59.21940100961365 ], "command_buffer_throughput_us_p50": 59.55078268016223, "command_buffer_throughput_us_p90": 60.313411995593924, "compute_latency_us": [ 46.834, 49.916, 49.666, 50.666, 49.708, 49.75, 50.542, 47.916, 49.75 ], "compute_latency_us_p50": 49.75, "compute_latency_us_p90": 50.5668, "command_buffer_latency_us": [ 46.8750367872417, 49.875001423060894, 49.62499951943755, 50.62500713393092, 49.666676204651594, 49.749971367418766, 50.541653763502836, 47.874986194074154, 49.749971367418766 ], "command_buffer_latency_us_p50": 49.749971367418766, "command_buffer_latency_us_p90": 50.55832443758845, "counter_control_throughput_ratio": 2.0565018942939592, "counter_control_latency_ratio": 1.0025183244169558 }, "download_ms": 0.70925, "allocation_upload_ms": 0.632542, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-0unc_rxv/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "0d8a82e58cbe97c43d15ef559cf043ad575745e0d3fb0f4f40ed60b2caca21c0", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.945167, "capture_ms": 0.071791, "compile_ms": 49.942375, "allocation_upload_ms": 1.797208, "cold_call_ms": 5.191333, "warmup_ms": 200.139917, "download_ms": 0.924, "repetitions": 1712, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 14.4546098131, 14.140478972, 14.3589853972, 13.9424649533, 13.9541962617, 13.9026483645, 14.1664228972, 14.3365946262, 14.1946553738 ], "latency_us": [ 320.917, 248.541, 253.375, 263.958, 258.75, 261.25, 325.542, 280.042, 251 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 880666, "compute_span_ns": 880666, "calibration_cpu_ns": 1333459, "calibration_gpu_ticks": 1333459, "compute_passes": 1, "command_buffer_ns": 880708.335899, "command_buffers": 2 }, { "compute_ns": 887541, "compute_span_ns": 887541, "calibration_cpu_ns": 1337917, "calibration_gpu_ticks": 1337917, "compute_passes": 1, "command_buffer_ns": 887541.682459, "command_buffers": 2 }, { "compute_ns": 936250, "compute_span_ns": 936250, "calibration_cpu_ns": 1424917, "calibration_gpu_ticks": 1424917, "compute_passes": 1, "command_buffer_ns": 936250.027735, "command_buffers": 2 }, { "compute_ns": 866417, "compute_span_ns": 866417, "calibration_cpu_ns": 1365125, "calibration_gpu_ticks": 1365125, "compute_passes": 1, "command_buffer_ns": 866541.580763, "command_buffers": 2 }, { "compute_ns": 851250, "compute_span_ns": 851250, "calibration_cpu_ns": 1383125, "calibration_gpu_ticks": 1383125, "compute_passes": 1, "command_buffer_ns": 851250.020787, "command_buffers": 2 }, { "compute_ns": 917334, "compute_span_ns": 917334, "calibration_cpu_ns": 1390834, "calibration_gpu_ticks": 1390834, "compute_passes": 1, "command_buffer_ns": 917416.648008, "command_buffers": 2 }, { "compute_ns": 844667, "compute_span_ns": 844667, "calibration_cpu_ns": 1334459, "calibration_gpu_ticks": 1334459, "compute_passes": 1, "command_buffer_ns": 844791.706186, "command_buffers": 2 }, { "compute_ns": 914708, "compute_span_ns": 914708, "calibration_cpu_ns": 1408708, "calibration_gpu_ticks": 1408708, "compute_passes": 1, "command_buffer_ns": 914708.361961, "command_buffers": 2 }, { "compute_ns": 868583, "compute_span_ns": 868583, "calibration_cpu_ns": 1347208, "calibration_gpu_ticks": 1347208, "compute_passes": 1, "command_buffer_ns": 868666.684255, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18000, "compute_span_ns": 18000, "calibration_cpu_ns": 454750, "calibration_gpu_ticks": 454750, "compute_passes": 1, "command_buffer_ns": 18000.0206456, "command_buffers": 2 }, { "compute_ns": 18250, "compute_span_ns": 18250, "calibration_cpu_ns": 453417, "calibration_gpu_ticks": 453417, "compute_passes": 1, "command_buffer_ns": 18291.6992344, "command_buffers": 2 }, { "compute_ns": 17917, "compute_span_ns": 17917, "calibration_cpu_ns": 455708, "calibration_gpu_ticks": 455708, "compute_passes": 1, "command_buffer_ns": 18000.0206456, "command_buffers": 2 }, { "compute_ns": 17875, "compute_span_ns": 17875, "calibration_cpu_ns": 405625, "calibration_gpu_ticks": 405625, "compute_passes": 1, "command_buffer_ns": 17875.0487976, "command_buffers": 2 }, { "compute_ns": 18041, "compute_span_ns": 18041, "calibration_cpu_ns": 424459, "calibration_gpu_ticks": 424459, "compute_passes": 1, "command_buffer_ns": 18083.374016, "command_buffers": 2 }, { "compute_ns": 18083, "compute_span_ns": 18083, "calibration_cpu_ns": 448042, "calibration_gpu_ticks": 448042, "compute_passes": 1, "command_buffer_ns": 18124.9342859, "command_buffers": 2 }, { "compute_ns": 18000, "compute_span_ns": 18000, "calibration_cpu_ns": 457042, "calibration_gpu_ticks": 457042, "compute_passes": 1, "command_buffer_ns": 18083.2576007, "command_buffers": 2 }, { "compute_ns": 19042, "compute_span_ns": 19042, "calibration_cpu_ns": 593875, "calibration_gpu_ticks": 593875, "compute_passes": 1, "command_buffer_ns": 19041.7049453, "command_buffers": 2 }, { "compute_ns": 18000, "compute_span_ns": 18000, "calibration_cpu_ns": 411209, "calibration_gpu_ticks": 411209, "compute_passes": 1, "command_buffer_ns": 18000.0206456, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 891458.359547, "command_buffers": 2 }, { "command_buffer_ns": 855416.699778, "command_buffers": 2 }, { "command_buffer_ns": 933625.036851, "command_buffers": 2 }, { "command_buffer_ns": 843583.373353, "command_buffers": 2 }, { "command_buffer_ns": 865750.014782, "command_buffers": 2 }, { "command_buffer_ns": 861416.62905, "command_buffers": 2 }, { "command_buffer_ns": 849624.97931, "command_buffers": 2 }, { "command_buffer_ns": 864458.270371, "command_buffers": 2 }, { "command_buffer_ns": 857249.950059, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 15583.3549798, "command_buffers": 2 }, { "command_buffer_ns": 18208.2876563, "command_buffers": 2 }, { "command_buffer_ns": 18208.345864, "command_buffers": 2 }, { "command_buffer_ns": 17958.3439603, "command_buffers": 2 }, { "command_buffer_ns": 18458.3477676, "command_buffers": 2 }, { "command_buffer_ns": 18041.6391231, "command_buffers": 2 }, { "command_buffer_ns": 18458.3477676, "command_buffers": 2 }, { "command_buffer_ns": 18291.6410267, "command_buffers": 2 }, { "command_buffer_ns": 18041.6973308, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 13.929036867921875, 13.365885934031251, 14.587891200796875, 13.180990208640626, 13.52734398096875, 13.45963482890625, 13.27539030171875, 13.507160474546875, 13.394530469671876 ], "command_buffer_throughput_us_p50": 13.45963482890625, "command_buffer_throughput_us_p90": 14.060807734496874, "command_buffer_latency_us": [ 15.583354979800001, 18.208287656299998, 18.208345863999998, 17.958343960300002, 18.4583477676, 18.0416391231, 18.4583477676, 18.2916410267, 18.0416973308 ], "command_buffer_latency_us_p50": 18.208287656299998, "command_buffer_latency_us_p90": 18.4583477676 }, "compute_throughput_us": [ 13.76040625, 13.867828125, 14.62890625, 13.537765625, 13.30078125, 14.33334375, 13.197921875, 14.2923125, 13.571609375 ], "compute_throughput_us_p50": 13.76040625, "compute_throughput_us_p90": 14.392456249999999, "command_buffer_throughput_us": [ 13.761067748421876, 13.867838788421876, 14.628906683359375, 13.539712199421874, 13.300781574796874, 14.334635125125, 13.19987040915625, 14.292318155640626, 13.572916941484374 ], "command_buffer_throughput_us_p50": 13.761067748421876, "command_buffer_throughput_us_p90": 14.393489436771876, "compute_latency_us": [ 18.0, 18.25, 17.917, 17.875, 18.041, 18.083, 18.0, 19.042, 18.0 ], "compute_latency_us_p50": 18.0, "compute_latency_us_p90": 18.4084, "command_buffer_latency_us": [ 18.000020645600003, 18.2916992344, 18.000020645600003, 17.8750487976, 18.083374016, 18.1249342859, 18.083257600699998, 19.0417049453, 18.000020645600003 ], "command_buffer_latency_us_p50": 18.083257600699998, "command_buffer_latency_us_p90": 18.44170037658, "counter_control_throughput_ratio": 1.0133178592722158, "counter_control_latency_ratio": 0.9976899798042366 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 14.1664228972, "throughput_us_p90": 14.37811028038, "latency_us_p50": 261.25, "latency_us_p90": 321.842 }, "slowdown": 0.42343450139980404, "device_compute_slowdown": 0.2914747863086621, "valid": true, "round": 2, "variant": "reference" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-jjutyw6u/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "8bb581805080afef9f47b3147742261e73f0bbc7f1d60afd69631b609a5e3f17", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.314667, "capture_ms": 0.078333, "compile_ms": 50.031875, "allocation_upload_ms": 5.362875, "cold_call_ms": 5.581625, "warmup_ms": 200.21175, "download_ms": 3.084541, "repetitions": 568, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 51.7043714789, 51.3982552817, 51.7032711268, 51.7450123239, 52.0239137324, 51.8711848592, 51.9851091549, 51.6663732394, 52.0373380282 ], "latency_us": [ 297.958, 271.125, 257.333, 275.625, 287.542, 263.458, 261.459, 284.5, 247.584 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3366375, "compute_span_ns": 3366375, "calibration_cpu_ns": 3802375, "calibration_gpu_ticks": 3802375, "compute_passes": 1, "command_buffer_ns": 3366458.34614, "command_buffers": 2 }, { "compute_ns": 3294459, "compute_span_ns": 3294459, "calibration_cpu_ns": 3750666, "calibration_gpu_ticks": 3750666, "compute_passes": 1, "command_buffer_ns": 3294541.67513, "command_buffers": 2 }, { "compute_ns": 3238041, "compute_span_ns": 3238041, "calibration_cpu_ns": 3693666, "calibration_gpu_ticks": 3693666, "compute_passes": 1, "command_buffer_ns": 3238083.38726, "command_buffers": 2 }, { "compute_ns": 3282083, "compute_span_ns": 3282083, "calibration_cpu_ns": 3645250, "calibration_gpu_ticks": 3645250, "compute_passes": 1, "command_buffer_ns": 3282208.28669, "command_buffers": 2 }, { "compute_ns": 3247041, "compute_span_ns": 3247041, "calibration_cpu_ns": 3683500, "calibration_gpu_ticks": 3683500, "compute_passes": 1, "command_buffer_ns": 3247083.33937, "command_buffers": 2 }, { "compute_ns": 3271500, "compute_span_ns": 3271500, "calibration_cpu_ns": 3684125, "calibration_gpu_ticks": 3684125, "compute_passes": 1, "command_buffer_ns": 3271583.35131, "command_buffers": 2 }, { "compute_ns": 3218292, "compute_span_ns": 3218292, "calibration_cpu_ns": 3671167, "calibration_gpu_ticks": 3671167, "compute_passes": 1, "command_buffer_ns": 3218333.35329, "command_buffers": 2 }, { "compute_ns": 3217333, "compute_span_ns": 3217333, "calibration_cpu_ns": 3638417, "calibration_gpu_ticks": 3638417, "compute_passes": 1, "command_buffer_ns": 3217375.02236, "command_buffers": 2 }, { "compute_ns": 3325167, "compute_span_ns": 3325167, "calibration_cpu_ns": 3742875, "calibration_gpu_ticks": 3742875, "compute_passes": 1, "command_buffer_ns": 3325166.7046, "command_buffers": 2 } ], "latency": [ { "compute_ns": 56708, "compute_span_ns": 56708, "calibration_cpu_ns": 511334, "calibration_gpu_ticks": 511334, "compute_passes": 1, "command_buffer_ns": 56791.6431464, "command_buffers": 2 }, { "compute_ns": 64375, "compute_span_ns": 64375, "calibration_cpu_ns": 509500, "calibration_gpu_ticks": 509500, "compute_passes": 1, "command_buffer_ns": 64416.6138954, "command_buffers": 2 }, { "compute_ns": 58833, "compute_span_ns": 58833, "calibration_cpu_ns": 466125, "calibration_gpu_ticks": 466125, "compute_passes": 1, "command_buffer_ns": 58916.6884311, "command_buffers": 2 }, { "compute_ns": 56208, "compute_span_ns": 56208, "calibration_cpu_ns": 419958, "calibration_gpu_ticks": 419958, "compute_passes": 1, "command_buffer_ns": 56249.962654, "command_buffers": 2 }, { "compute_ns": 57208, "compute_span_ns": 57208, "calibration_cpu_ns": 516334, "calibration_gpu_ticks": 516334, "compute_passes": 1, "command_buffer_ns": 57250.0284761, "command_buffers": 2 }, { "compute_ns": 65750, "compute_span_ns": 65750, "calibration_cpu_ns": 541000, "calibration_gpu_ticks": 541000, "compute_passes": 1, "command_buffer_ns": 65791.5952615, "command_buffers": 2 }, { "compute_ns": 52542, "compute_span_ns": 52542, "calibration_cpu_ns": 457125, "calibration_gpu_ticks": 457125, "compute_passes": 1, "command_buffer_ns": 52666.6408405, "command_buffers": 2 }, { "compute_ns": 56708, "compute_span_ns": 56708, "calibration_cpu_ns": 446833, "calibration_gpu_ticks": 446833, "compute_passes": 1, "command_buffer_ns": 56749.9082536, "command_buffers": 2 }, { "compute_ns": 61792, "compute_span_ns": 61792, "calibration_cpu_ns": 617084, "calibration_gpu_ticks": 617084, "compute_passes": 1, "command_buffer_ns": 61875.0345893, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3390583.38432, "command_buffers": 2 }, { "command_buffer_ns": 3272958.33267, "command_buffers": 2 }, { "command_buffer_ns": 3260583.38396, "command_buffers": 2 }, { "command_buffer_ns": 3277666.63395, "command_buffers": 2 }, { "command_buffer_ns": 3272291.62212, "command_buffers": 2 }, { "command_buffer_ns": 3252291.64446, "command_buffers": 2 }, { "command_buffer_ns": 3228500.01976, "command_buffers": 2 }, { "command_buffer_ns": 3276708.36123, "command_buffers": 2 }, { "command_buffer_ns": 3274374.99072, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 52875.0242665, "command_buffers": 2 }, { "command_buffer_ns": 64874.9992251, "command_buffers": 2 }, { "command_buffer_ns": 56166.7256989, "command_buffers": 2 }, { "command_buffer_ns": 57249.9702685, "command_buffers": 2 }, { "command_buffer_ns": 53125.0261702, "command_buffers": 2 }, { "command_buffer_ns": 64583.378844, "command_buffers": 2 }, { "command_buffer_ns": 55166.6598767, "command_buffers": 2 }, { "command_buffer_ns": 53083.349485, "command_buffers": 2 }, { "command_buffer_ns": 52499.9923073, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 52.977865380000004, 51.13997394796875, 50.946615374375, 51.213541155468754, 51.129556595625, 50.8170569446875, 50.44531280875, 51.19856814421875, 51.16210923 ], "command_buffer_throughput_us_p50": 51.13997394796875, "command_buffer_throughput_us_p90": 51.566406000375004, "command_buffer_latency_us": [ 52.8750242665, 64.8749992251, 56.1667256989, 57.2499702685, 53.1250261702, 64.583378844, 55.1666598767, 53.083349485, 52.499992307300005 ], "command_buffer_latency_us_p50": 55.1666598767, "command_buffer_latency_us_p90": 64.64170292022 }, "compute_throughput_us": [ 52.599609375, 51.475921875, 50.594390625, 51.282546875, 50.735015625, 51.1171875, 50.2858125, 50.270828125, 51.955734375 ], "compute_throughput_us_p50": 51.1171875, "compute_throughput_us_p90": 52.084509374999996, "command_buffer_throughput_us": [ 52.600911658437504, 51.47721367390625, 50.5950529259375, 51.28450447953125, 50.73567717765625, 51.118489864218745, 50.28645864515625, 50.271484724375, 51.955729759375 ], "command_buffer_throughput_us_p50": 51.118489864218745, "command_buffer_throughput_us_p90": 52.0847661391875, "compute_latency_us": [ 56.708, 64.375, 58.833, 56.208, 57.208, 65.75, 52.542, 56.708, 61.792 ], "compute_latency_us_p50": 57.208, "compute_latency_us_p90": 64.65, "command_buffer_latency_us": [ 56.7916431464, 64.4166138954, 58.916688431100006, 56.249962654, 57.2500284761, 65.79159526149999, 52.666640840499994, 56.7499082536, 61.8750345893 ], "command_buffer_latency_us_p50": 57.2500284761, "command_buffer_latency_us_p90": 64.69161016862, "counter_control_throughput_ratio": 0.996850962859602, "counter_control_latency_ratio": 1.0489607093520472 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 51.7450123239, "throughput_us_p90": 52.026598591559996, "latency_us_p50": 271.125, "latency_us_p90": 289.6252 }, "torch": { "cold_call_ms": 4.712459, "warmup_ms": 200.893875, "repetitions": 188, "throughput_us": [ 129.2247340425532, 128.2165319148936, 129.0159574468085, 128.6626755319149, 128.60172872340425, 128.82602127659572, 128.9033670212766, 128.81449468085106, 128.86059574468084 ], "latency_us": [ 610.875, 559.5830000000001, 408.08299999999997, 400.666, 376.25, 378.20799999999997, 380.083, 363.625, 392.792 ], "throughput_us_p50": 128.82602127659572, "throughput_us_p90": 129.05771276595743, "latency_us_p50": 392.792, "latency_us_p90": 569.8414, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7700416.666921228, "command_buffers": 11 }, { "command_buffer_ns": 7753416.779451072, "command_buffers": 11 }, { "command_buffer_ns": 7733583.392109722, "command_buffers": 11 }, { "command_buffer_ns": 7748999.982140958, "command_buffers": 11 }, { "command_buffer_ns": 7683083.473239094, "command_buffers": 11 }, { "command_buffer_ns": 7756708.306260407, "command_buffers": 11 }, { "command_buffer_ns": 7706499.833147973, "command_buffers": 11 }, { "command_buffer_ns": 7637499.9480322, "command_buffers": 11 }, { "command_buffer_ns": 7661291.630938649, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 139374.98442828655, "command_buffers": 1 }, { "command_buffer_ns": 121749.99574199319, "command_buffers": 1 }, { "command_buffer_ns": 123208.3304785192, "command_buffers": 1 }, { "command_buffer_ns": 138041.67974740267, "command_buffers": 1 }, { "command_buffer_ns": 125208.34570750594, "command_buffers": 1 }, { "command_buffer_ns": 128041.66181012988, "command_buffers": 1 }, { "command_buffer_ns": 136000.0460408628, "command_buffers": 1 }, { "command_buffer_ns": 121833.34911242127, "command_buffers": 1 }, { "command_buffer_ns": 128666.63746535778, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 120.31901042064419, 121.147137178923, 120.8372405017144, 121.07812472095247, 120.04817926936084, 121.19856728531886, 120.41405989293708, 119.33593668800313, 119.7076817334164 ], "command_buffer_throughput_us_p50": 120.41405989293708, "command_buffer_throughput_us_p90": 121.15742320020217, "command_buffer_latency_us": [ 139.37498442828655, 121.74999574199319, 123.2083304785192, 138.04167974740267, 125.20834570750594, 128.04166181012988, 136.0000460408628, 121.83334911242127, 128.66663746535778 ], "command_buffer_latency_us_p50": 128.04166181012988, "command_buffer_latency_us_p90": 138.30834068357944 }, "throughput": [ { "compute_ns": 8080335.0, "compute_span_ns": 9630042.0, "command_buffer_ns": 9386083.402205259, "calibration_cpu_ns": 10474583.0, "calibration_gpu_ticks": 10474583.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8160040.0, "compute_span_ns": 9751500.0, "command_buffer_ns": 9502083.237748593, "calibration_cpu_ns": 10481292.0, "calibration_gpu_ticks": 10481292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8180372.0, "compute_span_ns": 9674083.0, "command_buffer_ns": 9430083.27158168, "calibration_cpu_ns": 10332292.0, "calibration_gpu_ticks": 10332292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8165379.0, "compute_span_ns": 9715125.0, "command_buffer_ns": 9450250.130612403, "calibration_cpu_ns": 10442667.0, "calibration_gpu_ticks": 10442667.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8111957.0, "compute_span_ns": 9652375.0, "command_buffer_ns": 9399166.621733457, "calibration_cpu_ns": 11048209.0, "calibration_gpu_ticks": 11048209.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8129953.0, "compute_span_ns": 9637958.0, "command_buffer_ns": 9390041.755978018, "calibration_cpu_ns": 10307834.0, "calibration_gpu_ticks": 10307834.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8133249.0, "compute_span_ns": 9685000.0, "command_buffer_ns": 9435166.663024575, "calibration_cpu_ns": 10446500.0, "calibration_gpu_ticks": 10446500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8257953.0, "compute_span_ns": 9799667.0, "command_buffer_ns": 9554749.995004386, "calibration_cpu_ns": 10518042.0, "calibration_gpu_ticks": 10518042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8186126.0, "compute_span_ns": 9754333.0, "command_buffer_ns": 9510708.44894275, "calibration_cpu_ns": 10583708.0, "calibration_gpu_ticks": 10583708.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 135708.0, "compute_span_ns": 135708.0, "command_buffer_ns": 135708.3092443645, "calibration_cpu_ns": 683916.0, "calibration_gpu_ticks": 683916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 128292.0, "compute_span_ns": 128292.0, "command_buffer_ns": 128291.66371375322, "calibration_cpu_ns": 606583.0, "calibration_gpu_ticks": 606583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 130959.0, "compute_span_ns": 130959.0, "command_buffer_ns": 131000.00796839595, "calibration_cpu_ns": 674541.0, "calibration_gpu_ticks": 674541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138292.0, "compute_span_ns": 138292.0, "command_buffer_ns": 138333.35833624005, "calibration_cpu_ns": 625166.0, "calibration_gpu_ticks": 625166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 132042.0, "compute_span_ns": 132042.0, "command_buffer_ns": 132083.3107456565, "calibration_cpu_ns": 652125.0, "calibration_gpu_ticks": 652125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 119625.0, "compute_span_ns": 119625.0, "command_buffer_ns": 119625.00866502523, "calibration_cpu_ns": 598625.0, "calibration_gpu_ticks": 598625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 144875.0, "compute_span_ns": 144875.0, "command_buffer_ns": 144875.0263080001, "calibration_cpu_ns": 677750.0, "calibration_gpu_ticks": 677750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 133375.0, "compute_span_ns": 133375.0, "command_buffer_ns": 133374.99694898725, "calibration_cpu_ns": 588167.0, "calibration_gpu_ticks": 588167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 133209.0, "compute_span_ns": 133209.0, "command_buffer_ns": 133250.02510100603, "calibration_cpu_ns": 629375.0, "calibration_gpu_ticks": 629375.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 126.255234375, 127.500625, 127.8183125, 127.584046875, 126.749328125, 127.030515625, 127.082015625, 129.030515625, 127.90821875 ], "compute_throughput_us_p50": 127.500625, "compute_throughput_us_p90": 128.132678125, "command_buffer_throughput_us": [ 146.65755315945717, 148.47005058982177, 147.34505111846374, 147.6601582908188, 146.86197846458526, 146.71940243715653, 147.42447910975898, 149.29296867194353, 148.60481951473048 ], "command_buffer_throughput_us_p50": 147.42447910975898, "command_buffer_throughput_us_p90": 148.7424493461731, "compute_latency_us": [ 135.708, 128.292, 130.959, 138.292, 132.042, 119.625, 144.875, 133.375, 133.209 ], "compute_latency_us_p50": 133.209, "compute_latency_us_p90": 139.6086, "command_buffer_latency_us": [ 135.7083092443645, 128.29166371375322, 131.00000796839595, 138.33335833624005, 132.0833107456565, 119.62500866502523, 144.8750263080001, 133.37499694898725, 133.25002510100603 ], "command_buffer_latency_us_p50": 133.25002510100603, "command_buffer_latency_us_p90": 139.64169193059206, "counter_control_throughput_ratio": 1.2233586494890551, "counter_control_latency_ratio": 1.0537303342960507 }, "download_ms": 0.6765, "allocation_upload_ms": 16.453292, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.4016658421267311, "device_compute_slowdown": 0.4009171523669002, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "softmax", "m": 1024, "n": 4096, "k": 1 }, "name": "softmax_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.544458, "warmup_ms": 201.32975, "repetitions": 227, "throughput_us": [ 127.64959911894273, 128.267436123348, 129.652718061674, 127.11711013215859, 127.72742290748899, 128.22118502202642, 128.2628502202643, 127.7778986784141, 128.52220704845814 ], "latency_us": [ 416.208, 372.25, 413.584, 398.584, 474.208, 388.5, 368.16700000000003, 370.333, 372.625 ], "throughput_us_p50": 128.22118502202642, "throughput_us_p90": 128.74830925110132, "latency_us_p50": 388.5, "latency_us_p90": 427.80800000000005, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7649916.806258261, "command_buffers": 11 }, { "command_buffer_ns": 7725500.036031008, "command_buffers": 11 }, { "command_buffer_ns": 7768791.692797095, "command_buffers": 11 }, { "command_buffer_ns": 7664000.033400953, "command_buffers": 11 }, { "command_buffer_ns": 7729083.416052163, "command_buffers": 11 }, { "command_buffer_ns": 7681875.08219853, "command_buffers": 11 }, { "command_buffer_ns": 7794916.62979126, "command_buffers": 11 }, { "command_buffer_ns": 7813124.975655228, "command_buffers": 11 }, { "command_buffer_ns": 7752625.038847327, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 131833.30884203315, "command_buffers": 1 }, { "command_buffer_ns": 123000.0052601099, "command_buffers": 1 }, { "command_buffer_ns": 126500.0319108367, "command_buffers": 1 }, { "command_buffer_ns": 123750.01097097993, "command_buffers": 1 }, { "command_buffer_ns": 145833.2990296185, "command_buffers": 1 }, { "command_buffer_ns": 137166.6439808905, "command_buffers": 1 }, { "command_buffer_ns": 142583.33249017596, "command_buffers": 1 }, { "command_buffer_ns": 127000.03571808338, "command_buffers": 1 }, { "command_buffer_ns": 125125.05054473877, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 119.52995009778533, 120.7109380629845, 121.3873701999546, 119.75000052188989, 120.76692837581504, 120.02929815935204, 121.79557234048843, 122.08007774461294, 121.13476623198949 ], "command_buffer_throughput_us_p50": 120.76692837581504, "command_buffer_throughput_us_p90": 121.85247342131333, "command_buffer_latency_us": [ 131.83330884203315, 123.0000052601099, 126.5000319108367, 123.75001097097993, 145.8332990296185, 137.1666439808905, 142.58333249017596, 127.00003571808338, 125.12505054473877 ], "command_buffer_latency_us_p50": 127.00003571808338, "command_buffer_latency_us_p90": 143.23332579806447 }, "throughput": [ { "compute_ns": 8008166.0, "compute_span_ns": 9495541.0, "command_buffer_ns": 9262624.836992472, "calibration_cpu_ns": 10368375.0, "calibration_gpu_ticks": 10368375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8043290.0, "compute_span_ns": 9473417.0, "command_buffer_ns": 9237583.260983229, "calibration_cpu_ns": 10382292.0, "calibration_gpu_ticks": 10382292.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 7949253.0, "compute_span_ns": 9464334.0, "command_buffer_ns": 9235083.358362317, "calibration_cpu_ns": 10233875.0, "calibration_gpu_ticks": 10233875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8067002.0, "compute_span_ns": 9513333.0, "command_buffer_ns": 9297041.804529727, "calibration_cpu_ns": 10325708.0, "calibration_gpu_ticks": 10325708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8156919.0, "compute_span_ns": 9619125.0, "command_buffer_ns": 9414666.623342782, "calibration_cpu_ns": 10359375.0, "calibration_gpu_ticks": 10359375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8118873.0, "compute_span_ns": 9698875.0, "command_buffer_ns": 9451624.937355518, "calibration_cpu_ns": 10370375.0, "calibration_gpu_ticks": 10370375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8111586.0, "compute_span_ns": 9679459.0, "command_buffer_ns": 9433916.711714119, "calibration_cpu_ns": 10509625.0, "calibration_gpu_ticks": 10509625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8136915.0, "compute_span_ns": 9589166.0, "command_buffer_ns": 9375541.529152542, "calibration_cpu_ns": 10467125.0, "calibration_gpu_ticks": 10467125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8055881.0, "compute_span_ns": 9589250.0, "command_buffer_ns": 9342833.422124386, "calibration_cpu_ns": 10417750.0, "calibration_gpu_ticks": 10417750.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 128208.0, "compute_span_ns": 128208.0, "command_buffer_ns": 128208.31034332514, "calibration_cpu_ns": 739291.0, "calibration_gpu_ticks": 739291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 118125.0, "compute_span_ns": 118125.0, "command_buffer_ns": 118124.99724328518, "calibration_cpu_ns": 629333.0, "calibration_gpu_ticks": 629333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 118708.0, "compute_span_ns": 118708.0, "command_buffer_ns": 118666.6777357459, "calibration_cpu_ns": 636833.0, "calibration_gpu_ticks": 636833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125542.0, "compute_span_ns": 125542.0, "command_buffer_ns": 125541.70098155737, "calibration_cpu_ns": 723417.0, "calibration_gpu_ticks": 723417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 122333.0, "compute_span_ns": 122333.0, "command_buffer_ns": 122333.35291966796, "calibration_cpu_ns": 673333.0, "calibration_gpu_ticks": 673333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125834.0, "compute_span_ns": 125834.0, "command_buffer_ns": 125874.99804794788, "calibration_cpu_ns": 659292.0, "calibration_gpu_ticks": 659292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 131708.0, "compute_span_ns": 131708.0, "command_buffer_ns": 131708.33699405193, "calibration_cpu_ns": 672625.0, "calibration_gpu_ticks": 672625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125375.0, "compute_span_ns": 125375.0, "command_buffer_ns": 125374.9942407012, "calibration_cpu_ns": 642584.0, "calibration_gpu_ticks": 642584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 125167.0, "compute_span_ns": 125167.0, "command_buffer_ns": 125166.6690222919, "calibration_cpu_ns": 1253583.0, "calibration_gpu_ticks": 1253583.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 125.12759375, 125.67640625, 124.207078125, 126.04690625, 127.451859375, 126.857390625, 126.74353125, 127.139296875, 125.873140625 ], "compute_throughput_us_p50": 126.04690625, "compute_throughput_us_p90": 127.201809375, "command_buffer_throughput_us": [ 144.72851307800738, 144.33723845286295, 144.2981774744112, 145.26627819577698, 147.10416598973097, 147.68163964617997, 147.4049486205331, 146.49283639300847, 145.98177222069353 ], "command_buffer_throughput_us_p50": 145.98177222069353, "command_buffer_throughput_us_p90": 147.46028682566248, "compute_latency_us": [ 128.208, 118.125, 118.708, 125.542, 122.333, 125.834, 131.708, 125.375, 125.167 ], "compute_latency_us_p50": 125.375, "compute_latency_us_p90": 128.908, "command_buffer_latency_us": [ 128.20831034332514, 118.12499724328518, 118.6666777357459, 125.54170098155737, 122.33335291966796, 125.87499804794788, 131.70833699405193, 125.3749942407012, 125.1666690222919 ], "command_buffer_latency_us_p50": 125.3749942407012, "command_buffer_latency_us_p90": 128.9083156734705, "counter_control_throughput_ratio": 1.2102652484644663, "counter_control_latency_ratio": 0.9603657901760616 }, "download_ms": 0.761667, "allocation_upload_ms": 0.768042, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 9.74135319392784e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-_t_r10cx/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "117627a7b60c4cee3b9bce9542bc5f280f025f1ad809d52b8d22cb3cdd24d0d1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.267834, "capture_ms": 0.068666, "compile_ms": 49.338375, "allocation_upload_ms": 5.51575, "cold_call_ms": 8.873334, "warmup_ms": 200.266292, "download_ms": 4.445292, "repetitions": 523, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 54.4209674952, 53.9874130019, 54.1991720841, 53.8557992352, 53.6169521989, 53.8590669216, 54.1387017208, 53.7034741874, 54.3857552581 ], "latency_us": [ 406.125, 319.667, 318.375, 286.667, 404.625, 314.584, 290.417, 308.667, 298.583 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 16384, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3556375, "compute_span_ns": 3556375, "calibration_cpu_ns": 4044292, "calibration_gpu_ticks": 4044292, "compute_passes": 1, "command_buffer_ns": 3556499.95618, "command_buffers": 2 }, { "compute_ns": 3556292, "compute_span_ns": 3556292, "calibration_cpu_ns": 4053292, "calibration_gpu_ticks": 4053292, "compute_passes": 1, "command_buffer_ns": 3556333.36585, "command_buffers": 2 }, { "compute_ns": 3455042, "compute_span_ns": 3455042, "calibration_cpu_ns": 3925625, "calibration_gpu_ticks": 3925625, "compute_passes": 1, "command_buffer_ns": 3455083.29337, "command_buffers": 2 }, { "compute_ns": 3454292, "compute_span_ns": 3454292, "calibration_cpu_ns": 3971958, "calibration_gpu_ticks": 3971958, "compute_passes": 1, "command_buffer_ns": 3454416.64103, "command_buffers": 2 }, { "compute_ns": 3524625, "compute_span_ns": 3524625, "calibration_cpu_ns": 3992209, "calibration_gpu_ticks": 3992209, "compute_passes": 1, "command_buffer_ns": 3524666.71029, "command_buffers": 2 }, { "compute_ns": 3503417, "compute_span_ns": 3503417, "calibration_cpu_ns": 4104417, "calibration_gpu_ticks": 4104417, "compute_passes": 1, "command_buffer_ns": 3503458.34158, "command_buffers": 2 }, { "compute_ns": 3467500, "compute_span_ns": 3467500, "calibration_cpu_ns": 3910083, "calibration_gpu_ticks": 3910083, "compute_passes": 1, "command_buffer_ns": 3467583.33035, "command_buffers": 2 }, { "compute_ns": 3525000, "compute_span_ns": 3525000, "calibration_cpu_ns": 4001834, "calibration_gpu_ticks": 4001834, "compute_passes": 1, "command_buffer_ns": 3525000.00736, "command_buffers": 2 }, { "compute_ns": 3430833, "compute_span_ns": 3430833, "calibration_cpu_ns": 3917875, "calibration_gpu_ticks": 3917875, "compute_passes": 1, "command_buffer_ns": 3430958.3134, "command_buffers": 2 } ], "latency": [ { "compute_ns": 68208, "compute_span_ns": 68208, "calibration_cpu_ns": 513875, "calibration_gpu_ticks": 513875, "compute_passes": 1, "command_buffer_ns": 68208.319135, "command_buffers": 2 }, { "compute_ns": 56500, "compute_span_ns": 56500, "calibration_cpu_ns": 471875, "calibration_gpu_ticks": 471875, "compute_passes": 1, "command_buffer_ns": 56500.0227652, "command_buffers": 2 }, { "compute_ns": 55833, "compute_span_ns": 55833, "calibration_cpu_ns": 455584, "calibration_gpu_ticks": 455584, "compute_passes": 1, "command_buffer_ns": 55916.6073799, "command_buffers": 2 }, { "compute_ns": 54708, "compute_span_ns": 54708, "calibration_cpu_ns": 512209, "calibration_gpu_ticks": 512209, "compute_passes": 1, "command_buffer_ns": 54708.3327547, "command_buffers": 2 }, { "compute_ns": 66208, "compute_span_ns": 66208, "calibration_cpu_ns": 530750, "calibration_gpu_ticks": 530750, "compute_passes": 1, "command_buffer_ns": 66250.0387989, "command_buffers": 2 }, { "compute_ns": 54959, "compute_span_ns": 54959, "calibration_cpu_ns": 443833, "calibration_gpu_ticks": 443833, "compute_passes": 1, "command_buffer_ns": 55083.3647139, "command_buffers": 2 }, { "compute_ns": 57292, "compute_span_ns": 57292, "calibration_cpu_ns": 476500, "calibration_gpu_ticks": 476500, "compute_passes": 1, "command_buffer_ns": 57333.3818465, "command_buffers": 2 }, { "compute_ns": 56500, "compute_span_ns": 56500, "calibration_cpu_ns": 514125, "calibration_gpu_ticks": 514125, "compute_passes": 1, "command_buffer_ns": 56500.0227652, "command_buffers": 2 }, { "compute_ns": 70042, "compute_span_ns": 70042, "calibration_cpu_ns": 453833, "calibration_gpu_ticks": 453833, "compute_passes": 1, "command_buffer_ns": 70166.6576788, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3545250.04514, "command_buffers": 2 }, { "command_buffer_ns": 3444541.71136, "command_buffers": 2 }, { "command_buffer_ns": 3487291.68763, "command_buffers": 2 }, { "command_buffer_ns": 3442583.37282, "command_buffers": 2 }, { "command_buffer_ns": 3416875.02805, "command_buffers": 2 }, { "command_buffer_ns": 3499833.34308, "command_buffers": 2 }, { "command_buffer_ns": 3457541.69393, "command_buffers": 2 }, { "command_buffer_ns": 3450999.96775, "command_buffers": 2 }, { "command_buffer_ns": 3464333.36381, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 66958.3096169, "command_buffers": 2 }, { "command_buffer_ns": 55000.0113435, "command_buffers": 2 }, { "command_buffer_ns": 55833.3704248, "command_buffers": 2 }, { "command_buffer_ns": 54916.6579731, "command_buffers": 2 }, { "command_buffer_ns": 70458.27806, "command_buffers": 2 }, { "command_buffer_ns": 56624.9946132, "command_buffers": 2 }, { "command_buffer_ns": 55499.9569431, "command_buffers": 2 }, { "command_buffer_ns": 55458.3384655, "command_buffers": 2 }, { "command_buffer_ns": 74666.691944, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 55.3945319553125, 53.82096424, 54.48893261921875, 53.7903652003125, 53.38867231328125, 54.684895985625, 54.024088967656255, 53.92187449609375, 54.13020880953125 ], "command_buffer_throughput_us_p50": 54.024088967656255, "command_buffer_throughput_us_p90": 54.826823179562496, "command_buffer_latency_us": [ 66.95830961690001, 55.0000113435, 55.8333704248, 54.9166579731, 70.45827806, 56.624994613199995, 55.499956943099995, 55.4583384655, 74.66669194400001 ], "command_buffer_latency_us_p50": 55.8333704248, "command_buffer_latency_us_p90": 71.2999608368 }, "compute_throughput_us": [ 55.568359375, 55.5670625, 53.98503125, 53.9733125, 55.072265625, 54.740890625, 54.1796875, 55.078125, 53.606765625 ], "compute_throughput_us_p50": 54.740890625, "compute_throughput_us_p90": 55.567321875, "command_buffer_throughput_us": [ 55.5703118153125, 55.56770884140625, 53.98567645890625, 53.97526001609375, 55.072917348281244, 54.7415365871875, 54.18098953671875, 55.078125115000006, 53.608723646875 ], "command_buffer_throughput_us_p50": 54.7415365871875, "command_buffer_throughput_us_p90": 55.5682294361875, "compute_latency_us": [ 68.208, 56.5, 55.833, 54.708, 66.208, 54.959, 57.292, 56.5, 70.042 ], "compute_latency_us_p50": 56.5, "compute_latency_us_p90": 68.5748, "command_buffer_latency_us": [ 68.208319135, 56.5000227652, 55.9166073799, 54.7083327547, 66.25003879890001, 55.0833647139, 57.333381846500004, 56.5000227652, 70.1666576788 ], "command_buffer_latency_us_p50": 56.5000227652, "command_buffer_latency_us_p90": 68.59998684375999, "counter_control_throughput_ratio": 1.003173234862636, "counter_control_latency_ratio": 1.0014908101457374 }, "correctness": { "max_abs_error": 1.2343096304160556e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 53.9874130019, "throughput_us_p90": 54.392797705519996, "latency_us_p50": 314.584, "latency_us_p90": 404.925 }, "slowdown": 0.4210490878915664, "device_compute_slowdown": 0.43428983902569995, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-csh4nehf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "e81310ef6c7b3435af776cabf332187d02f0ddc801b9234c32af8a56116f248b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.890791, "capture_ms": 0.064667, "compile_ms": 59.162875, "allocation_upload_ms": 1.5205, "cold_call_ms": 2.111292, "warmup_ms": 200.187709, "download_ms": 0.28575, "repetitions": 5680, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.84880422535, 4.74087429577, 4.43324524648, 4.71177376761, 4.58392024648, 4.79148327465, 4.64053697183, 4.72248380282, 4.75699823944 ], "latency_us": [ 411.125, 198.542, 178, 175.042, 184.666, 356.167, 214.875, 209.833, 203.333 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 48, "normalized_kernel_cost": 48, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 321333, "compute_span_ns": 321333, "calibration_cpu_ns": 750792, "calibration_gpu_ticks": 750792, "compute_passes": 1, "command_buffer_ns": 321375.03149, "command_buffers": 2 }, { "compute_ns": 336333, "compute_span_ns": 336333, "calibration_cpu_ns": 799709, "calibration_gpu_ticks": 799709, "compute_passes": 1, "command_buffer_ns": 336374.971084, "command_buffers": 2 }, { "compute_ns": 343250, "compute_span_ns": 343250, "calibration_cpu_ns": 775250, "calibration_gpu_ticks": 775250, "compute_passes": 1, "command_buffer_ns": 343249.99433, "command_buffers": 2 }, { "compute_ns": 330625, "compute_span_ns": 330625, "calibration_cpu_ns": 703833, "calibration_gpu_ticks": 703833, "compute_passes": 1, "command_buffer_ns": 330624.985509, "command_buffers": 2 }, { "compute_ns": 265791, "compute_span_ns": 265791, "calibration_cpu_ns": 745708, "calibration_gpu_ticks": 745708, "compute_passes": 1, "command_buffer_ns": 265833.339654, "command_buffers": 2 }, { "compute_ns": 290167, "compute_span_ns": 290167, "calibration_cpu_ns": 779167, "calibration_gpu_ticks": 779167, "compute_passes": 1, "command_buffer_ns": 290291.674901, "command_buffers": 2 }, { "compute_ns": 290958, "compute_span_ns": 290958, "calibration_cpu_ns": 700667, "calibration_gpu_ticks": 700667, "compute_passes": 1, "command_buffer_ns": 290999.945719, "command_buffers": 2 }, { "compute_ns": 339084, "compute_span_ns": 339084, "calibration_cpu_ns": 732333, "calibration_gpu_ticks": 732333, "compute_passes": 1, "command_buffer_ns": 339208.345395, "command_buffers": 2 }, { "compute_ns": 298334, "compute_span_ns": 298334, "calibration_cpu_ns": 711750, "calibration_gpu_ticks": 711750, "compute_passes": 1, "command_buffer_ns": 298416.707665, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6792, "compute_span_ns": 6792, "calibration_cpu_ns": 388209, "calibration_gpu_ticks": 388209, "compute_passes": 1, "command_buffer_ns": 6958.26020092, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 441125, "calibration_gpu_ticks": 441125, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 360458, "calibration_gpu_ticks": 360458, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 464708, "calibration_gpu_ticks": 464708, "compute_passes": 1, "command_buffer_ns": 7166.70183465, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 369917, "calibration_gpu_ticks": 369917, "compute_passes": 1, "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "compute_ns": 6917, "compute_span_ns": 6917, "calibration_cpu_ns": 343292, "calibration_gpu_ticks": 343292, "compute_passes": 1, "command_buffer_ns": 6916.64172336, "command_buffers": 2 }, { "compute_ns": 14333, "compute_span_ns": 14333, "calibration_cpu_ns": 340208, "calibration_gpu_ticks": 340208, "compute_passes": 1, "command_buffer_ns": 14374.9639392, "command_buffers": 2 }, { "compute_ns": 7042, "compute_span_ns": 7042, "calibration_cpu_ns": 359125, "calibration_gpu_ticks": 359125, "compute_passes": 1, "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "compute_ns": 7083, "compute_span_ns": 7083, "calibration_cpu_ns": 364042, "calibration_gpu_ticks": 364042, "compute_passes": 1, "command_buffer_ns": 7125.02514943, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 308624.992613, "command_buffers": 2 }, { "command_buffer_ns": 318083.330058, "command_buffers": 2 }, { "command_buffer_ns": 324999.971781, "command_buffers": 2 }, { "command_buffer_ns": 264458.30008, "command_buffers": 2 }, { "command_buffer_ns": 273250.043392, "command_buffers": 2 }, { "command_buffer_ns": 282291.613985, "command_buffers": 2 }, { "command_buffer_ns": 293458.346277, "command_buffers": 2 }, { "command_buffer_ns": 310083.385557, "command_buffers": 2 }, { "command_buffer_ns": 290875.032078, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7166.64362699, "command_buffers": 2 }, { "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "command_buffer_ns": 7083.34846422, "command_buffers": 2 }, { "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "command_buffer_ns": 7458.26400816, "command_buffers": 2 }, { "command_buffer_ns": 7333.35036784, "command_buffers": 2 }, { "command_buffer_ns": 7333.29216018, "command_buffers": 2 }, { "command_buffer_ns": 7375.02705306, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.822265509578124, 4.97005203215625, 5.078124559078125, 4.13216093875, 4.269531928, 4.410806468515625, 4.585286660578125, 4.845052899328125, 4.54492237621875 ], "command_buffer_throughput_us_p50": 4.585286660578125, "command_buffer_throughput_us_p90": 4.991666537540625, "command_buffer_latency_us": [ 7.16664362699, 7.1249669417699995, 7.08334846422, 7.208320312200001, 7.208320312200001, 7.4582640081600005, 7.3333503678400005, 7.33329216018, 7.37502705306 ], "command_buffer_latency_us_p50": 7.208320312200001, "command_buffer_latency_us_p90": 7.39167444408 }, "compute_throughput_us": [ 5.020828125, 5.255203125, 5.36328125, 5.166015625, 4.152984375, 4.533859375, 4.54621875, 5.2981875, 4.66146875 ], "compute_throughput_us_p50": 5.020828125, "compute_throughput_us_p90": 5.31120625, "command_buffer_throughput_us": [ 5.02148486703125, 5.255858923187501, 5.36328116140625, 5.166015398578125, 4.15364593209375, 4.5358074203281245, 4.546874151859376, 5.300130396796876, 4.662761057265625 ], "command_buffer_throughput_us_p50": 5.02148486703125, "command_buffer_throughput_us_p90": 5.3127605497187504, "compute_latency_us": [ 6.792, 7.042, 7.0, 7.042, 7.125, 6.917, 14.333, 7.042, 7.083 ], "compute_latency_us_p50": 7.042, "compute_latency_us_p90": 8.566600000000001, "command_buffer_latency_us": [ 6.95826020092, 7.208320312200001, 6.99999509379, 7.1667018346499995, 7.12502514943, 6.91664172336, 14.3749639392, 7.1249669417699995, 7.12502514943 ], "command_buffer_latency_us_p50": 7.12502514943, "command_buffer_latency_us_p90": 8.641649037600002, "counter_control_throughput_ratio": 1.0413123991321984, "counter_control_latency_ratio": 0.9882324904879322 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.72248380282, "throughput_us_p90": 4.80294746479, "latency_us_p50": 203.333, "latency_us_p90": 367.1586 }, "torch": { "cold_call_ms": 3.192167, "warmup_ms": 200.080583, "repetitions": 3099, "throughput_us": [ 8.063958696353662, 8.14070409809616, 7.5278315585672795, 7.219613746369796, 7.3845730880929334, 7.489579864472411, 7.422878347854146, 7.743290738948048, 7.851565020974508 ], "latency_us": [ 280.917, 216.667, 220.458, 228.959, 217.625, 224.5, 229.042, 235.875, 229.333 ], "throughput_us_p50": 7.5278315585672795, "throughput_us_p90": 8.079307776702162, "latency_us_p50": 228.959, "latency_us_p90": 244.8834, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 364208.30292627215, "command_buffers": 1 }, { "command_buffer_ns": 347083.3180472255, "command_buffers": 1 }, { "command_buffer_ns": 354416.6684150696, "command_buffers": 1 }, { "command_buffer_ns": 289708.37593078613, "command_buffers": 1 }, { "command_buffer_ns": 303041.71377792954, "command_buffers": 1 }, { "command_buffer_ns": 290166.64484515786, "command_buffers": 1 }, { "command_buffer_ns": 289625.02256035805, "command_buffers": 1 }, { "command_buffer_ns": 304333.3417735994, "command_buffers": 1 }, { "command_buffer_ns": 289625.02256035805, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 8041.621185839176, "command_buffers": 1 }, { "command_buffer_ns": 8125.032763928175, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 8000.002708286047, "command_buffers": 1 }, { "command_buffer_ns": 8166.651241481304, "command_buffers": 1 }, { "command_buffer_ns": 8083.297871053219, "command_buffers": 1 }, { "command_buffer_ns": 8124.9745562672615, "command_buffers": 1 }, { "command_buffer_ns": 8125.032763928175, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 5.690754733223002, 5.423176844487898, 5.537760443985462, 4.526693373918533, 4.735026777780149, 4.533853825705592, 4.5253909775055945, 4.75520846521249, 4.5253909775055945 ], "command_buffer_throughput_us_p50": 4.735026777780149, "command_buffer_throughput_us_p90": 5.56835930183297, "command_buffer_latency_us": [ 8.041621185839176, 8.125032763928175, 8.000002708286047, 8.124974556267262, 8.000002708286047, 8.166651241481304, 8.083297871053219, 8.124974556267262, 8.125032763928175 ], "command_buffer_latency_us_p50": 8.124974556267262, "command_buffer_latency_us_p90": 8.1333564594388 }, "throughput": [ { "compute_ns": 356042.0, "compute_span_ns": 356042.0, "command_buffer_ns": 356083.3283700049, "calibration_cpu_ns": 960042.0, "calibration_gpu_ticks": 960042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 368667.0, "compute_span_ns": 368667.0, "command_buffer_ns": 368708.3371914923, "calibration_cpu_ns": 963042.0, "calibration_gpu_ticks": 963042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 368167.0, "compute_span_ns": 368167.0, "command_buffer_ns": 368166.6566990316, "calibration_cpu_ns": 956000.0, "calibration_gpu_ticks": 956000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 304000.0, "compute_span_ns": 304000.0, "command_buffer_ns": 303999.98649954796, "calibration_cpu_ns": 862500.0, "calibration_gpu_ticks": 862500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 304333.0, "compute_span_ns": 304333.0, "command_buffer_ns": 304333.2835659385, "calibration_cpu_ns": 873833.0, "calibration_gpu_ticks": 873833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 310166.0, "compute_span_ns": 310166.0, "command_buffer_ns": 310125.0040344894, "calibration_cpu_ns": 942708.0, "calibration_gpu_ticks": 942708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 288708.0, "compute_span_ns": 288708.0, "command_buffer_ns": 288666.6334234178, "calibration_cpu_ns": 881459.0, "calibration_gpu_ticks": 881459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 303584.0, "compute_span_ns": 303584.0, "command_buffer_ns": 303625.0127479434, "calibration_cpu_ns": 904708.0, "calibration_gpu_ticks": 904708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 290667.0, "compute_span_ns": 290667.0, "command_buffer_ns": 290666.64865240455, "calibration_cpu_ns": 942084.0, "calibration_gpu_ticks": 942084.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8124.9745562672615, "calibration_cpu_ns": 407583.0, "calibration_gpu_ticks": 407583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8250.0, "compute_span_ns": 8250.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 493875.0, "calibration_gpu_ticks": 493875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8292.0, "compute_span_ns": 8292.0, "command_buffer_ns": 8333.357982337475, "calibration_cpu_ns": 428833.0, "calibration_gpu_ticks": 428833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8124.9745562672615, "calibration_cpu_ns": 393208.0, "calibration_gpu_ticks": 393208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8124.9745562672615, "calibration_cpu_ns": 385833.0, "calibration_gpu_ticks": 385833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8250.0, "compute_span_ns": 8250.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 488750.0, "calibration_gpu_ticks": 488750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8666.0, "compute_span_ns": 8666.0, "command_buffer_ns": 8624.978363513947, "calibration_cpu_ns": 442625.0, "calibration_gpu_ticks": 442625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8167.0, "compute_span_ns": 8167.0, "command_buffer_ns": 8166.651241481304, "calibration_cpu_ns": 387792.0, "calibration_gpu_ticks": 387792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8209.0, "compute_span_ns": 8209.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 387917.0, "calibration_gpu_ticks": 387917.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 5.56315625, 5.760421875, 5.752609375, 4.75, 4.755203125, 4.84634375, 4.5110625, 4.7435, 4.541671875 ], "compute_throughput_us_p50": 4.755203125, "compute_throughput_us_p90": 5.754171875, "command_buffer_throughput_us": [ 5.563802005781326, 5.7610677686170675, 5.752604010922369, 4.749999789055437, 4.755207555717789, 4.845703188038897, 4.510416147240903, 4.744140824186616, 4.541666385193821 ], "command_buffer_throughput_us_p50": 4.755207555717789, "command_buffer_throughput_us_p90": 5.754296762461308, "compute_latency_us": [ 8.125, 8.25, 8.292, 8.125, 8.125, 8.25, 8.666, 8.167, 8.209 ], "compute_latency_us_p50": 8.209, "compute_latency_us_p90": 8.3668, "command_buffer_latency_us": [ 8.124974556267262, 8.25000461190939, 8.333357982337475, 8.124974556267262, 8.124974556267262, 8.25000461190939, 8.624978363513947, 8.166651241481304, 8.25000461190939 ], "command_buffer_latency_us_p50": 8.25000461190939, "command_buffer_latency_us_p90": 8.39168205857277, "counter_control_throughput_ratio": 1.0042620198120824, "counter_control_latency_ratio": 1.0153810884967798 }, "download_ms": 1.072875, "allocation_upload_ms": 0.934792, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.6273365398891574, "device_compute_slowdown": 1.0558598640305192, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 37, "n": 1537, "k": 1 }, "name": "rmsnorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.26, "warmup_ms": 200.290542, "repetitions": 3452, "throughput_us": [ 8.332319235225956, 8.013458285052144, 8.499746523754345, 8.147861239860951, 8.139303302433373, 8.034701911935109, 7.650540845886442, 7.330388180764774, 8.011997682502898 ], "latency_us": [ 233.084, 244.708, 180.333, 226.458, 222.75, 217.333, 226.5, 240.209, 306.5 ], "throughput_us_p50": 8.034701911935109, "throughput_us_p90": 8.365804692931635, "latency_us_p50": 226.5, "latency_us_p90": 257.0664, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 369458.34290236235, "command_buffers": 1 }, { "command_buffer_ns": 360874.98301640153, "command_buffers": 1 }, { "command_buffer_ns": 371833.33188295364, "command_buffers": 1 }, { "command_buffer_ns": 367166.6490845382, "command_buffers": 1 }, { "command_buffer_ns": 370249.9670907855, "command_buffers": 1 }, { "command_buffer_ns": 374249.997548759, "command_buffers": 1 }, { "command_buffer_ns": 366874.97049570084, "command_buffers": 1 }, { "command_buffer_ns": 368541.63045063615, "command_buffers": 1 }, { "command_buffer_ns": 370124.9952428043, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 8041.621185839176, "command_buffers": 1 }, { "command_buffer_ns": 8208.327926695347, "command_buffers": 1 }, { "command_buffer_ns": 8250.00461190939, "command_buffers": 1 }, { "command_buffer_ns": 8624.978363513947, "command_buffers": 1 }, { "command_buffer_ns": 8083.356078714132, "command_buffers": 1 }, { "command_buffer_ns": 8375.034667551517, "command_buffers": 1 }, { "command_buffer_ns": 8250.00461190939, "command_buffers": 1 }, { "command_buffer_ns": 11041.644029319286, "command_buffers": 1 }, { "command_buffer_ns": 8750.008419156075, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 5.772786607849412, 5.638671609631274, 5.809895810671151, 5.73697889194591, 5.7851557357935235, 5.847656211699359, 5.732421413995326, 5.75846297579119, 5.783203050668817 ], "command_buffer_throughput_us_p50": 5.772786607849412, "command_buffer_throughput_us_p90": 5.817447890876792, "command_buffer_latency_us": [ 8.041621185839176, 8.208327926695347, 8.25000461190939, 8.624978363513947, 8.083356078714132, 8.375034667551517, 8.25000461190939, 11.041644029319286, 8.750008419156075 ], "command_buffer_latency_us_p50": 8.25000461190939, "command_buffer_latency_us_p90": 9.208335541188717 }, "throughput": [ { "compute_ns": 366291.0, "compute_span_ns": 366291.0, "command_buffer_ns": 366249.99484047294, "calibration_cpu_ns": 938333.0, "calibration_gpu_ticks": 938333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 371584.0, "compute_span_ns": 371584.0, "command_buffer_ns": 371625.00666454434, "calibration_cpu_ns": 873459.0, "calibration_gpu_ticks": 873459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 363125.0, "compute_span_ns": 363125.0, "command_buffer_ns": 363125.0001490116, "calibration_cpu_ns": 951750.0, "calibration_gpu_ticks": 951750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 375250.0, "compute_span_ns": 375250.0, "command_buffer_ns": 375250.00516325235, "calibration_cpu_ns": 944375.0, "calibration_gpu_ticks": 944375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 376625.0, "compute_span_ns": 376625.0, "command_buffer_ns": 376624.9865293503, "calibration_cpu_ns": 953041.0, "calibration_gpu_ticks": 953041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 366708.0, "compute_span_ns": 366708.0, "command_buffer_ns": 366708.3219625056, "calibration_cpu_ns": 929542.0, "calibration_gpu_ticks": 929542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 374834.0, "compute_span_ns": 374834.0, "command_buffer_ns": 374875.0314116478, "calibration_cpu_ns": 917792.0, "calibration_gpu_ticks": 917792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 348042.0, "compute_span_ns": 348042.0, "command_buffer_ns": 348041.6489765048, "calibration_cpu_ns": 922834.0, "calibration_gpu_ticks": 922834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 363125.0, "compute_span_ns": 363125.0, "command_buffer_ns": 363125.0001490116, "calibration_cpu_ns": 915875.0, "calibration_gpu_ticks": 915875.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 8292.0, "compute_span_ns": 8292.0, "command_buffer_ns": 8333.357982337475, "calibration_cpu_ns": 391791.0, "calibration_gpu_ticks": 391791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8292.0, "compute_span_ns": 8292.0, "command_buffer_ns": 8291.681297123432, "calibration_cpu_ns": 367375.0, "calibration_gpu_ticks": 367375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8167.0, "compute_span_ns": 8167.0, "command_buffer_ns": 8208.327926695347, "calibration_cpu_ns": 390541.0, "calibration_gpu_ticks": 390541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8125.0, "compute_span_ns": 8125.0, "command_buffer_ns": 8125.032763928175, "calibration_cpu_ns": 408334.0, "calibration_gpu_ticks": 408334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8333.0, "compute_span_ns": 8333.0, "command_buffer_ns": 8333.299774676561, "calibration_cpu_ns": 373291.0, "calibration_gpu_ticks": 373291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8208.0, "compute_span_ns": 8208.0, "command_buffer_ns": 8166.651241481304, "calibration_cpu_ns": 374791.0, "calibration_gpu_ticks": 374791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8041.0, "compute_span_ns": 8041.0, "command_buffer_ns": 8000.002708286047, "calibration_cpu_ns": 406458.0, "calibration_gpu_ticks": 406458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 11416.0, "compute_span_ns": 11416.0, "command_buffer_ns": 11374.999303370714, "calibration_cpu_ns": 456375.0, "calibration_gpu_ticks": 456375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 8291.0, "compute_span_ns": 8291.0, "command_buffer_ns": 8250.00461190939, "calibration_cpu_ns": 439959.0, "calibration_gpu_ticks": 439959.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 5.723296875, 5.806, 5.673828125, 5.86328125, 5.884765625, 5.7298125, 5.85678125, 5.43815625, 5.673828125 ], "compute_throughput_us_p50": 5.7298125, "compute_throughput_us_p90": 5.867578125, "command_buffer_throughput_us": [ 5.72265616938239, 5.806640729133505, 5.673828127328306, 5.863281330675818, 5.884765414521098, 5.72981753066415, 5.857422365806997, 5.4381507652578875, 5.673828127328306 ], "command_buffer_throughput_us_p50": 5.72981753066415, "command_buffer_throughput_us_p90": 5.867578147444874, "compute_latency_us": [ 8.292, 8.292, 8.167, 8.125, 8.333, 8.208, 8.041, 11.416, 8.291 ], "compute_latency_us_p50": 8.291, "compute_latency_us_p90": 8.9496, "command_buffer_latency_us": [ 8.333357982337475, 8.291681297123432, 8.208327926695347, 8.125032763928175, 8.333299774676561, 8.166651241481304, 8.000002708286047, 11.374999303370714, 8.25000461190939 ], "command_buffer_latency_us_p50": 8.25000461190939, "command_buffer_latency_us_p90": 8.941686246544123, "counter_control_throughput_ratio": 0.9913160762951365, "counter_control_latency_ratio": 0.9949482834041232 }, "download_ms": 0.283875, "allocation_upload_ms": 0.612333, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.807831588873654e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-myi3cl_k/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "54d99166e42515c7bfff1cac173ad46750fa91f9c92e668c07d69318fdc050bf", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.1195, "capture_ms": 0.063084, "compile_ms": 50.7655, "allocation_upload_ms": 1.194083, "cold_call_ms": 1.773, "warmup_ms": 200.020166, "download_ms": 0.287041, "repetitions": 7183, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.13619532229, 4.17425978004, 4.17098245858, 4.09549157734, 4.34383266045, 4.81698106641, 4.47215063344, 4.08889043575, 4.12546415147 ], "latency_us": [ 281.333, 223.417, 197.625, 206.917, 209.625, 205.458, 201.25, 218.959, 239.208 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 24, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12296, "global_write_bytes": 6148, "private_read_bytes": 18444, "private_write_bytes": 6148 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 64, "global_write_bytes": 32, "private_read_bytes": 96, "private_write_bytes": 32 }, "striped_storage_scalars_per_worker": 8, "reduction_operations": 1, "reduction_elements": 1537, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 56, "normalized_kernel_cost": 56, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 3074, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 246208, "compute_span_ns": 246208, "calibration_cpu_ns": 1049917, "calibration_gpu_ticks": 1049917, "compute_passes": 1, "command_buffer_ns": 246249.954216, "command_buffers": 2 }, { "compute_ns": 267792, "compute_span_ns": 267792, "calibration_cpu_ns": 718250, "calibration_gpu_ticks": 718250, "compute_passes": 1, "command_buffer_ns": 267833.354883, "command_buffers": 2 }, { "compute_ns": 257875, "compute_span_ns": 257875, "calibration_cpu_ns": 660666, "calibration_gpu_ticks": 660666, "compute_passes": 1, "command_buffer_ns": 257916.632108, "command_buffers": 2 }, { "compute_ns": 246166, "compute_span_ns": 246166, "calibration_cpu_ns": 665167, "calibration_gpu_ticks": 665167, "compute_passes": 1, "command_buffer_ns": 246208.335739, "command_buffers": 2 }, { "compute_ns": 265666, "compute_span_ns": 265666, "calibration_cpu_ns": 696292, "calibration_gpu_ticks": 696292, "compute_passes": 1, "command_buffer_ns": 265666.632913, "command_buffers": 2 }, { "compute_ns": 261875, "compute_span_ns": 261875, "calibration_cpu_ns": 711875, "calibration_gpu_ticks": 711875, "compute_passes": 1, "command_buffer_ns": 261999.957729, "command_buffers": 2 }, { "compute_ns": 246333, "compute_span_ns": 246333, "calibration_cpu_ns": 687083, "calibration_gpu_ticks": 687083, "compute_passes": 1, "command_buffer_ns": 246291.689109, "command_buffers": 2 }, { "compute_ns": 267583, "compute_span_ns": 267583, "calibration_cpu_ns": 717000, "calibration_gpu_ticks": 717000, "compute_passes": 1, "command_buffer_ns": 267624.971457, "command_buffers": 2 }, { "compute_ns": 267334, "compute_span_ns": 267334, "calibration_cpu_ns": 670417, "calibration_gpu_ticks": 670417, "compute_passes": 1, "command_buffer_ns": 267458.322924, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6750, "compute_span_ns": 6750, "calibration_cpu_ns": 432458, "calibration_gpu_ticks": 432458, "compute_passes": 1, "command_buffer_ns": 6833.28835294, "command_buffers": 2 }, { "compute_ns": 6708, "compute_span_ns": 6708, "calibration_cpu_ns": 379250, "calibration_gpu_ticks": 379250, "compute_passes": 1, "command_buffer_ns": 6791.61166772, "command_buffers": 2 }, { "compute_ns": 7166, "compute_span_ns": 7166, "calibration_cpu_ns": 524291, "calibration_gpu_ticks": 524291, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 6791, "compute_span_ns": 6791, "calibration_cpu_ns": 460500, "calibration_gpu_ticks": 460500, "compute_passes": 1, "command_buffer_ns": 6791.66987538, "command_buffers": 2 }, { "compute_ns": 6750, "compute_span_ns": 6750, "calibration_cpu_ns": 413083, "calibration_gpu_ticks": 413083, "compute_passes": 1, "command_buffer_ns": 6749.99319017, "command_buffers": 2 }, { "compute_ns": 6792, "compute_span_ns": 6792, "calibration_cpu_ns": 396000, "calibration_gpu_ticks": 396000, "compute_passes": 1, "command_buffer_ns": 6916.69993103, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 401333, "calibration_gpu_ticks": 401333, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 374792, "calibration_gpu_ticks": 374792, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 365750, "calibration_gpu_ticks": 365750, "compute_passes": 1, "command_buffer_ns": 7208.37851986, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 271583.383437, "command_buffers": 2 }, { "command_buffer_ns": 287958.362605, "command_buffers": 2 }, { "command_buffer_ns": 246499.95612, "command_buffers": 2 }, { "command_buffer_ns": 246666.662861, "command_buffers": 2 }, { "command_buffer_ns": 267458.322924, "command_buffers": 2 }, { "command_buffer_ns": 245541.625191, "command_buffers": 2 }, { "command_buffer_ns": 246666.662861, "command_buffers": 2 }, { "command_buffer_ns": 267208.379228, "command_buffers": 2 }, { "command_buffer_ns": 246250.012424, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6916.64172336, "command_buffers": 2 }, { "command_buffer_ns": 6791.61166772, "command_buffers": 2 }, { "command_buffer_ns": 6499.93307889, "command_buffers": 2 }, { "command_buffer_ns": 6749.99319017, "command_buffers": 2 }, { "command_buffer_ns": 6791.66987538, "command_buffers": 2 }, { "command_buffer_ns": 6666.63981974, "command_buffers": 2 }, { "command_buffer_ns": 6874.96503815, "command_buffers": 2 }, { "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "command_buffer_ns": 6875.08145347, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.243490366203125, 4.499349415703125, 3.8515618143749997, 3.854166607203125, 4.179036295687499, 3.836587893609375, 3.854166607203125, 4.1751309254375, 3.8476564441249996 ], "command_buffer_throughput_us_p50": 3.854166607203125, "command_buffer_throughput_us_p90": 4.294662176103125, "command_buffer_latency_us": [ 6.91664172336, 6.79161166772, 6.49993307889, 6.7499931901700005, 6.79166987538, 6.66663981974, 6.87496503815, 6.99999509379, 6.87508145347 ], "command_buffer_latency_us_p50": 6.79166987538, "command_buffer_latency_us_p90": 6.933312397446 }, "compute_throughput_us": [ 3.847, 4.18425, 4.029296875, 3.84634375, 4.15103125, 4.091796875, 3.848953125, 4.180984375, 4.17709375 ], "compute_throughput_us_p50": 4.091796875, "compute_throughput_us_p90": 4.1816375, "command_buffer_throughput_us": [ 3.8476555346250003, 4.184896170046875, 4.0299473766875, 3.847005245921875, 4.151041139265625, 4.093749339515624, 3.848307642328125, 4.181640179015625, 4.179036295687499 ], "command_buffer_throughput_us_p50": 4.093749339515624, "command_buffer_throughput_us_p90": 4.182291377221875, "compute_latency_us": [ 6.75, 6.708, 7.166, 6.791, 6.75, 6.792, 6.958, 6.958, 7.125 ], "compute_latency_us_p50": 6.792, "compute_latency_us_p90": 7.1332, "command_buffer_latency_us": [ 6.8332883529399995, 6.79161166772, 7.208320312200001, 6.79166987538, 6.7499931901700005, 6.91669993103, 7.0416717790100005, 7.0416717790100005, 7.20837851986 ], "command_buffer_latency_us_p50": 6.91669993103, "command_buffer_latency_us_p90": 7.208331953732, "counter_control_throughput_ratio": 0.9984798361170869, "counter_control_latency_ratio": 1.0061743299638721 }, "correctness": { "max_abs_error": 1.5564224420394623e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.17098245858, "throughput_us_p90": 4.541116720034, "latency_us_p50": 209.625, "latency_us_p90": 247.633 }, "slowdown": 0.5191209959369164, "device_compute_slowdown": 0.7141240441986539, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-kx8a32he/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b5a32df1046f38df4f8e9cbba5277df92f4beb4d4f9631ac4564ef5cda134264", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 19.112875, "capture_ms": 0.067417, "compile_ms": 51.234791, "allocation_upload_ms": 2.005166, "cold_call_ms": 3.434416, "warmup_ms": 200.529792, "download_ms": 0.701125, "repetitions": 1850, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 15.2615313514, 15.4653151351, 15.4971848649, 15.6561259459, 15.5602475676, 15.4712610811, 14.7658108108, 15.2043243243, 14.5684459459 ], "latency_us": [ 310.458, 249.291, 237.958, 252.791, 259.958, 236.583, 248.958, 241.542, 259.292 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 68, "normalized_kernel_cost": 272, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 911667, "compute_span_ns": 911667, "calibration_cpu_ns": 1339042, "calibration_gpu_ticks": 1339042, "compute_passes": 1, "command_buffer_ns": 911750.015803, "command_buffers": 2 }, { "compute_ns": 910542, "compute_span_ns": 910542, "calibration_cpu_ns": 1329333, "calibration_gpu_ticks": 1329333, "compute_passes": 1, "command_buffer_ns": 910583.301447, "command_buffers": 2 }, { "compute_ns": 915834, "compute_span_ns": 915834, "calibration_cpu_ns": 1357000, "calibration_gpu_ticks": 1357000, "compute_passes": 1, "command_buffer_ns": 916000.048164, "command_buffers": 2 }, { "compute_ns": 921292, "compute_span_ns": 921292, "calibration_cpu_ns": 1399708, "calibration_gpu_ticks": 1399708, "compute_passes": 1, "command_buffer_ns": 921416.678466, "command_buffers": 2 }, { "compute_ns": 897833, "compute_span_ns": 897833, "calibration_cpu_ns": 1333250, "calibration_gpu_ticks": 1333250, "compute_passes": 1, "command_buffer_ns": 897833.378986, "command_buffers": 2 }, { "compute_ns": 931417, "compute_span_ns": 931417, "calibration_cpu_ns": 1341375, "calibration_gpu_ticks": 1341375, "compute_passes": 1, "command_buffer_ns": 931416.638196, "command_buffers": 2 }, { "compute_ns": 906375, "compute_span_ns": 906375, "calibration_cpu_ns": 1357083, "calibration_gpu_ticks": 1357083, "compute_passes": 1, "command_buffer_ns": 906375.003979, "command_buffers": 2 }, { "compute_ns": 902750, "compute_span_ns": 902750, "calibration_cpu_ns": 1319250, "calibration_gpu_ticks": 1319250, "compute_passes": 1, "command_buffer_ns": 902833.35885, "command_buffers": 2 }, { "compute_ns": 972959, "compute_span_ns": 972959, "calibration_cpu_ns": 1385583, "calibration_gpu_ticks": 1385583, "compute_passes": 1, "command_buffer_ns": 973041.635007, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18750, "compute_span_ns": 18750, "calibration_cpu_ns": 440166, "calibration_gpu_ticks": 440166, "compute_passes": 1, "command_buffer_ns": 18750.0263564, "command_buffers": 2 }, { "compute_ns": 18458, "compute_span_ns": 18458, "calibration_cpu_ns": 441625, "calibration_gpu_ticks": 441625, "compute_passes": 1, "command_buffer_ns": 18458.3477676, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 441916, "calibration_gpu_ticks": 441916, "compute_passes": 1, "command_buffer_ns": 18583.3778232, "command_buffers": 2 }, { "compute_ns": 18667, "compute_span_ns": 18667, "calibration_cpu_ns": 432708, "calibration_gpu_ticks": 432708, "compute_passes": 1, "command_buffer_ns": 18666.672986, "command_buffers": 2 }, { "compute_ns": 18708, "compute_span_ns": 18708, "calibration_cpu_ns": 427083, "calibration_gpu_ticks": 427083, "compute_passes": 1, "command_buffer_ns": 18791.644834, "command_buffers": 2 }, { "compute_ns": 18708, "compute_span_ns": 18708, "calibration_cpu_ns": 416041, "calibration_gpu_ticks": 416041, "compute_passes": 1, "command_buffer_ns": 18749.9099411, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 404625, "calibration_gpu_ticks": 404625, "compute_passes": 1, "command_buffer_ns": 18500.0244528, "command_buffers": 2 }, { "compute_ns": 18375, "compute_span_ns": 18375, "calibration_cpu_ns": 603875, "calibration_gpu_ticks": 603875, "compute_passes": 1, "command_buffer_ns": 18416.6128747, "command_buffers": 2 }, { "compute_ns": 18542, "compute_span_ns": 18542, "calibration_cpu_ns": 420500, "calibration_gpu_ticks": 420500, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 982249.970548, "command_buffers": 2 }, { "command_buffer_ns": 933333.300054, "command_buffers": 2 }, { "command_buffer_ns": 950333.313085, "command_buffers": 2 }, { "command_buffer_ns": 964250.00811, "command_buffers": 2 }, { "command_buffer_ns": 908749.992959, "command_buffers": 2 }, { "command_buffer_ns": 904874.992557, "command_buffers": 2 }, { "command_buffer_ns": 908833.288122, "command_buffers": 2 }, { "command_buffer_ns": 975666.684099, "command_buffers": 2 }, { "command_buffer_ns": 967166.677583, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 16416.6558534, "command_buffers": 2 }, { "command_buffer_ns": 18791.644834, "command_buffers": 2 }, { "command_buffer_ns": 18791.7030416, "command_buffers": 2 }, { "command_buffer_ns": 18708.3496712, "command_buffers": 2 }, { "command_buffer_ns": 18833.3797269, "command_buffers": 2 }, { "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "command_buffer_ns": 18416.6128747, "command_buffers": 2 }, { "command_buffer_ns": 18541.701138, "command_buffers": 2 }, { "command_buffer_ns": 19208.3534785, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.3476557898125, 14.58333281334375, 14.848958016953125, 15.066406376718751, 14.199218639984377, 14.138671758703126, 14.200520126906252, 15.244791939046873, 15.111979337234375 ], "command_buffer_throughput_us_p50": 14.848958016953125, "command_buffer_throughput_us_p90": 15.265364709199998, "command_buffer_latency_us": [ 16.416655853399998, 18.791644834, 18.7917030416, 18.7083496712, 18.8333797269, 18.624996300800003, 18.4166128747, 18.541701138, 19.2083534785 ], "command_buffer_latency_us_p50": 18.7083496712, "command_buffer_latency_us_p90": 18.90837447722 }, "compute_throughput_us": [ 14.244796875, 14.22721875, 14.30990625, 14.3951875, 14.028640625, 14.553390625, 14.162109375, 14.10546875, 15.202484375 ], "compute_throughput_us_p50": 14.244796875, "compute_throughput_us_p90": 14.683209375, "command_buffer_throughput_us": [ 14.246093996921875, 14.227864085109376, 14.312500752562501, 14.39713560103125, 14.02864654665625, 14.5533849718125, 14.162109437171875, 14.10677123203125, 15.203775546984375 ], "command_buffer_throughput_us_p50": 14.246093996921875, "command_buffer_throughput_us_p90": 14.683463086846874, "compute_latency_us": [ 18.75, 18.458, 18.5, 18.667, 18.708, 18.708, 18.5, 18.375, 18.542 ], "compute_latency_us_p50": 18.542, "compute_latency_us_p90": 18.7164, "command_buffer_latency_us": [ 18.750026356400003, 18.4583477676, 18.5833778232, 18.666672986000002, 18.791644834, 18.7499099411, 18.5000244528, 18.4166128747, 18.541642930400002 ], "command_buffer_latency_us_p50": 18.5833778232, "command_buffer_latency_us_p90": 18.75835005192, "counter_control_throughput_ratio": 0.9756250006233745, "counter_control_latency_ratio": 0.9977722949414315 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 15.4653151351, "throughput_us_p90": 15.579423243259999, "latency_us_p50": 249.291, "latency_us_p90": 270.05800000000005 }, "torch": { "cold_call_ms": 4.956625, "warmup_ms": 200.338083, "repetitions": 1325, "throughput_us": [ 19.827924528301885, 18.88364754716981, 20.668301886792452, 20.940503396226415, 20.95210716981132, 20.723993962264153, 19.04776679245283, 19.58877358490566, 19.56380528301887 ], "latency_us": [ 432.375, 249.583, 254.91599999999997, 247.25, 260.041, 269.875, 272.792, 257.41700000000003, 244.875 ], "throughput_us_p50": 19.827924528301885, "throughput_us_p90": 20.942824150943395, "latency_us_p50": 257.41700000000003, "latency_us_p90": 304.7086, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1069749.9965317547, "command_buffers": 1 }, { "command_buffer_ns": 1028458.3549946547, "command_buffers": 1 }, { "command_buffer_ns": 1093291.6775345802, "command_buffers": 1 }, { "command_buffer_ns": 1083125.0110641122, "command_buffers": 1 }, { "command_buffer_ns": 1029541.6577719152, "command_buffers": 1 }, { "command_buffer_ns": 1087791.6938625276, "command_buffers": 1 }, { "command_buffer_ns": 1031249.9944120646, "command_buffers": 1 }, { "command_buffer_ns": 1027916.6745021939, "command_buffers": 1 }, { "command_buffer_ns": 1082375.0053532422, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 18666.672986000776, "command_buffers": 1 }, { "command_buffer_ns": 21041.661966592073, "command_buffers": 1 }, { "command_buffer_ns": 21000.043489038944, "command_buffers": 1 }, { "command_buffer_ns": 20875.013433396816, "command_buffers": 1 }, { "command_buffer_ns": 20958.308596163988, "command_buffers": 1 }, { "command_buffer_ns": 20916.69011861086, "command_buffers": 1 }, { "command_buffer_ns": 21166.633814573288, "command_buffers": 1 }, { "command_buffer_ns": 21166.6920222342, "command_buffers": 1 }, { "command_buffer_ns": 21125.01533702016, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.714843695808668, 16.06966179679148, 17.082682461477816, 16.923828297876753, 16.086588402686175, 16.996745216601994, 16.11328116268851, 16.06119803909678, 16.91210945864441 ], "command_buffer_throughput_us_p50": 16.714843695808668, "command_buffer_throughput_us_p90": 17.01393266557716, "command_buffer_latency_us": [ 18.666672986000776, 21.041661966592073, 21.000043489038944, 20.875013433396816, 20.958308596163988, 20.91669011861086, 21.166633814573288, 21.1666920222342, 21.12501533702016 ], "command_buffer_latency_us_p50": 21.000043489038944, "command_buffer_latency_us_p90": 21.16664545610547 }, "throughput": [ { "compute_ns": 1086167.0, "compute_span_ns": 1086167.0, "command_buffer_ns": 1086208.3290703595, "calibration_cpu_ns": 1775084.0, "calibration_gpu_ticks": 1775084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1030750.0, "compute_span_ns": 1030750.0, "command_buffer_ns": 1030749.9906048179, "calibration_cpu_ns": 1761083.0, "calibration_gpu_ticks": 1761083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1029209.0, "compute_span_ns": 1029209.0, "command_buffer_ns": 1029249.9791830778, "calibration_cpu_ns": 1676041.0, "calibration_gpu_ticks": 1676041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1029292.0, "compute_span_ns": 1029292.0, "command_buffer_ns": 1029333.3325535059, "calibration_cpu_ns": 1629250.0, "calibration_gpu_ticks": 1629250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1030667.0, "compute_span_ns": 1030667.0, "command_buffer_ns": 1030708.3139196038, "calibration_cpu_ns": 1701708.0, "calibration_gpu_ticks": 1701708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1076833.0, "compute_span_ns": 1076833.0, "command_buffer_ns": 1076791.6683107615, "calibration_cpu_ns": 1791458.0, "calibration_gpu_ticks": 1791458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1067292.0, "compute_span_ns": 1067292.0, "command_buffer_ns": 1067291.6541807353, "calibration_cpu_ns": 1705916.0, "calibration_gpu_ticks": 1705916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1056625.0, "compute_span_ns": 1056625.0, "command_buffer_ns": 1056625.0421106815, "calibration_cpu_ns": 1716041.0, "calibration_gpu_ticks": 1716041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1030416.0, "compute_span_ns": 1030416.0, "command_buffer_ns": 1030374.9586455524, "calibration_cpu_ns": 1689458.0, "calibration_gpu_ticks": 1689458.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 20916.0, "compute_span_ns": 20916.0, "command_buffer_ns": 20875.013433396816, "calibration_cpu_ns": 492208.0, "calibration_gpu_ticks": 492208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21208.0, "compute_span_ns": 21208.0, "command_buffer_ns": 21166.633814573288, "calibration_cpu_ns": 503000.0, "calibration_gpu_ticks": 503000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20792.0, "compute_span_ns": 20792.0, "command_buffer_ns": 20833.27854052186, "calibration_cpu_ns": 443458.0, "calibration_gpu_ticks": 443458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21250.0, "compute_span_ns": 21250.0, "command_buffer_ns": 21249.987185001373, "calibration_cpu_ns": 432084.0, "calibration_gpu_ticks": 432084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20875.0, "compute_span_ns": 20875.0, "command_buffer_ns": 20875.013433396816, "calibration_cpu_ns": 495209.0, "calibration_gpu_ticks": 495209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21208.0, "compute_span_ns": 21208.0, "command_buffer_ns": 21166.6920222342, "calibration_cpu_ns": 434500.0, "calibration_gpu_ticks": 434500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21000.0, "compute_span_ns": 21000.0, "command_buffer_ns": 20999.98528137803, "calibration_cpu_ns": 472500.0, "calibration_gpu_ticks": 472500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21041.0, "compute_span_ns": 21041.0, "command_buffer_ns": 21000.043489038944, "calibration_cpu_ns": 538000.0, "calibration_gpu_ticks": 538000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21125.0, "compute_span_ns": 21125.0, "command_buffer_ns": 21125.01533702016, "calibration_cpu_ns": 461042.0, "calibration_gpu_ticks": 461042.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 16.971359375, 16.10546875, 16.081390625, 16.0826875, 16.104171875, 16.825515625, 16.6764375, 16.509765625, 16.10025 ], "compute_throughput_us_p50": 16.10546875, "compute_throughput_us_p90": 16.854684375, "command_buffer_throughput_us": [ 16.972005141724367, 16.10546860320028, 16.08203092473559, 16.08333332114853, 16.10481740499381, 16.824869817355648, 16.67643209657399, 16.5097662829794, 16.099608728836756 ], "command_buffer_throughput_us_p50": 16.10546860320028, "command_buffer_throughput_us_p90": 16.85429688222939, "compute_latency_us": [ 20.916, 21.208, 20.792, 21.25, 20.875, 21.208, 21.0, 21.041, 21.125 ], "compute_latency_us_p50": 21.041, "compute_latency_us_p90": 21.2164, "command_buffer_latency_us": [ 20.875013433396816, 21.166633814573288, 20.83327854052186, 21.249987185001373, 20.875013433396816, 21.1666920222342, 20.99998528137803, 21.000043489038944, 21.12501533702016 ], "command_buffer_latency_us_p50": 21.000043489038944, "command_buffer_latency_us_p90": 21.183351054787636, "counter_control_throughput_ratio": 1.001133180128149, "counter_control_latency_ratio": 1.0 }, "download_ms": 0.354167, "allocation_upload_ms": 1.399667, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7799764979448653, "device_compute_slowdown": 0.8844695609992724, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 256, "n": 3072, "k": 1 }, "name": "rmsnorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.306, "warmup_ms": 200.320042, "repetitions": 1253, "throughput_us": [ 20.716114924181962, 20.990090183559456, 20.869247406225057, 21.308592976855547, 20.321328810853952, 19.553870710295293, 19.007083000798083, 19.181597765363126, 19.477122106943337 ], "latency_us": [ 363.70799999999997, 294.333, 250.04199999999997, 247.875, 256.834, 282.5, 255.042, 245.458, 240.125 ], "throughput_us_p50": 20.321328810853952, "throughput_us_p90": 21.053790742218673, "latency_us_p50": 255.042, "latency_us_p90": 308.208, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1193666.7142435908, "command_buffers": 1 }, { "command_buffer_ns": 1049624.988809228, "command_buffers": 1 }, { "command_buffer_ns": 1124124.9740123749, "command_buffers": 1 }, { "command_buffer_ns": 1181583.3277069032, "command_buffers": 1 }, { "command_buffer_ns": 1168624.9636113644, "command_buffers": 1 }, { "command_buffer_ns": 1186958.3395309746, "command_buffers": 1 }, { "command_buffer_ns": 1256916.6719913483, "command_buffers": 1 }, { "command_buffer_ns": 1288999.9779872596, "command_buffers": 1 }, { "command_buffer_ns": 1218624.9950900674, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 18916.67488962412, "command_buffers": 1 }, { "command_buffer_ns": 20999.98528137803, "command_buffers": 1 }, { "command_buffer_ns": 21166.6920222342, "command_buffers": 1 }, { "command_buffer_ns": 21291.663870215416, "command_buffers": 1 }, { "command_buffer_ns": 21124.957129359245, "command_buffers": 1 }, { "command_buffer_ns": 21083.338651806116, "command_buffers": 1 }, { "command_buffer_ns": 21499.989088624716, "command_buffers": 1 }, { "command_buffer_ns": 21791.6676774621, "command_buffers": 1 }, { "command_buffer_ns": 21666.695829480886, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 18.651042410056107, 16.400390450144187, 17.564452718943357, 18.462239495420363, 18.259765056427568, 18.54622405517148, 19.639322999864817, 20.14062465605093, 19.041015548282303 ], "command_buffer_throughput_us_p50": 18.54622405517148, "command_buffer_throughput_us_p90": 19.73958333110204, "command_buffer_latency_us": [ 18.91667488962412, 20.99998528137803, 21.1666920222342, 21.291663870215416, 21.124957129359245, 21.083338651806116, 21.499989088624716, 21.7916676774621, 21.666695829480886 ], "command_buffer_latency_us_p50": 21.1666920222342, "command_buffer_latency_us_p90": 21.69169019907713 }, "throughput": [ { "compute_ns": 1111333.0, "compute_span_ns": 1111333.0, "command_buffer_ns": 1111291.698180139, "calibration_cpu_ns": 1698042.0, "calibration_gpu_ticks": 1698042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1028500.0, "compute_span_ns": 1028500.0, "command_buffer_ns": 1028499.9734722078, "calibration_cpu_ns": 1660959.0, "calibration_gpu_ticks": 1660959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1125667.0, "compute_span_ns": 1125667.0, "command_buffer_ns": 1125666.662119329, "calibration_cpu_ns": 1842291.0, "calibration_gpu_ticks": 1842291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1191583.0, "compute_span_ns": 1191583.0, "command_buffer_ns": 1191583.345644176, "calibration_cpu_ns": 1778042.0, "calibration_gpu_ticks": 1778042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1212208.0, "compute_span_ns": 1212208.0, "command_buffer_ns": 1212208.2989662886, "calibration_cpu_ns": 1799750.0, "calibration_gpu_ticks": 1799750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1210417.0, "compute_span_ns": 1210417.0, "command_buffer_ns": 1210458.343848586, "calibration_cpu_ns": 1833458.0, "calibration_gpu_ticks": 1833458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1197917.0, "compute_span_ns": 1197917.0, "command_buffer_ns": 1197916.6883975267, "calibration_cpu_ns": 1792792.0, "calibration_gpu_ticks": 1792792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1198000.0, "compute_span_ns": 1198000.0, "command_buffer_ns": 1198000.0417679548, "calibration_cpu_ns": 1811875.0, "calibration_gpu_ticks": 1811875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 1177417.0, "compute_span_ns": 1177417.0, "command_buffer_ns": 1177458.3254009485, "calibration_cpu_ns": 1786750.0, "calibration_gpu_ticks": 1786750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 18708.0, "compute_span_ns": 18708.0, "command_buffer_ns": 18666.614778339863, "calibration_cpu_ns": 447917.0, "calibration_gpu_ticks": 447917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 23708.0, "compute_span_ns": 23708.0, "command_buffer_ns": 23708.329536020756, "calibration_cpu_ns": 456125.0, "calibration_gpu_ticks": 456125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21000.0, "compute_span_ns": 21000.0, "command_buffer_ns": 20999.98528137803, "calibration_cpu_ns": 482167.0, "calibration_gpu_ticks": 482167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 20958.0, "compute_span_ns": 20958.0, "command_buffer_ns": 20916.631910949945, "calibration_cpu_ns": 396500.0, "calibration_gpu_ticks": 396500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21333.0, "compute_span_ns": 21333.0, "command_buffer_ns": 21291.663870215416, "calibration_cpu_ns": 480041.0, "calibration_gpu_ticks": 480041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21250.0, "compute_span_ns": 21250.0, "command_buffer_ns": 21249.987185001373, "calibration_cpu_ns": 587875.0, "calibration_gpu_ticks": 587875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 25167.0, "compute_span_ns": 25167.0, "command_buffer_ns": 25208.34095776081, "calibration_cpu_ns": 518292.0, "calibration_gpu_ticks": 518292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21125.0, "compute_span_ns": 21125.0, "command_buffer_ns": 21125.01533702016, "calibration_cpu_ns": 418084.0, "calibration_gpu_ticks": 418084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 21541.0, "compute_span_ns": 21541.0, "command_buffer_ns": 21500.04729628563, "calibration_cpu_ns": 449292.0, "calibration_gpu_ticks": 449292.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 17.364578125, 16.0703125, 17.588546875, 18.618484375, 18.94075, 18.912765625, 18.717453125, 18.71875, 18.397140625 ], "compute_throughput_us_p50": 18.618484375, "compute_throughput_us_p90": 18.9183625, "command_buffer_throughput_us": [ 17.363932784064673, 16.070312085503247, 17.588541595614515, 18.61848977569025, 18.94075467134826, 18.913411622634158, 18.717448256211355, 18.718750652624294, 18.39778633438982 ], "command_buffer_throughput_us_p50": 18.61848977569025, "command_buffer_throughput_us_p90": 18.918880232376978, "compute_latency_us": [ 18.708, 23.708, 21.0, 20.958, 21.333, 21.25, 25.167, 21.125, 21.541 ], "compute_latency_us_p50": 21.25, "compute_latency_us_p90": 23.9998, "command_buffer_latency_us": [ 18.666614778339863, 23.708329536020756, 20.99998528137803, 20.916631910949945, 21.291663870215416, 21.249987185001373, 25.20834095776081, 21.12501533702016, 21.50004729628563 ], "command_buffer_latency_us_p50": 21.249987185001373, "command_buffer_latency_us_p90": 24.008331820368767, "counter_control_throughput_ratio": 0.9798737496131966, "counter_control_latency_ratio": 0.9923085395896634 }, "download_ms": 0.352, "allocation_upload_ms": 1.170583, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8163651027691685e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-avjol3jh/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "a8c235e7c8be57d97298e1f9f70525f6c2c7f4ae4cee9647b15503a46aab5ade", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 16.5885, "capture_ms": 0.060166, "compile_ms": 53.663458, "allocation_upload_ms": 1.926417, "cold_call_ms": 5.3145, "warmup_ms": 200.368, "download_ms": 0.886542, "repetitions": 1715, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 15.6179055394, 15.6171527697, 15.7204326531, 15.6710396501, 15.4741498542, 14.8060979592, 14.9465014577, 15.8655492711, 14.906535277 ], "latency_us": [ 322.417, 295.5, 278.042, 270.875, 239.875, 280.167, 292, 257.208, 337.334 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 36, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 12288, "private_read_bytes": 36864, "private_write_bytes": 12288 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 48, "private_read_bytes": 144, "private_write_bytes": 48 }, "striped_storage_scalars_per_worker": 12, "reduction_operations": 1, "reduction_elements": 3072, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 60, "normalized_kernel_cost": 240, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 6144, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 973125, "compute_span_ns": 973125, "calibration_cpu_ns": 1460250, "calibration_gpu_ticks": 1460250, "compute_passes": 1, "command_buffer_ns": 973166.606855, "command_buffers": 2 }, { "compute_ns": 921542, "compute_span_ns": 921542, "calibration_cpu_ns": 1394208, "calibration_gpu_ticks": 1394208, "compute_passes": 1, "command_buffer_ns": 921666.68037, "command_buffers": 2 }, { "compute_ns": 993583, "compute_span_ns": 993583, "calibration_cpu_ns": 1468292, "calibration_gpu_ticks": 1468292, "compute_passes": 1, "command_buffer_ns": 993624.969851, "command_buffers": 2 }, { "compute_ns": 950125, "compute_span_ns": 950125, "calibration_cpu_ns": 1419208, "calibration_gpu_ticks": 1419208, "compute_passes": 1, "command_buffer_ns": 950208.399445, "command_buffers": 2 }, { "compute_ns": 918500, "compute_span_ns": 918500, "calibration_cpu_ns": 1382875, "calibration_gpu_ticks": 1382875, "compute_passes": 1, "command_buffer_ns": 918624.980841, "command_buffers": 2 }, { "compute_ns": 941375, "compute_span_ns": 941375, "calibration_cpu_ns": 1364333, "calibration_gpu_ticks": 1364333, "compute_passes": 1, "command_buffer_ns": 941499.951296, "command_buffers": 2 }, { "compute_ns": 908375, "compute_span_ns": 908375, "calibration_cpu_ns": 1363500, "calibration_gpu_ticks": 1363500, "compute_passes": 1, "command_buffer_ns": 908458.372578, "command_buffers": 2 }, { "compute_ns": 977917, "compute_span_ns": 977917, "calibration_cpu_ns": 1438250, "calibration_gpu_ticks": 1438250, "compute_passes": 1, "command_buffer_ns": 978041.673079, "command_buffers": 2 }, { "compute_ns": 1001041, "compute_span_ns": 1001041, "calibration_cpu_ns": 1441917, "calibration_gpu_ticks": 1441917, "compute_passes": 1, "command_buffer_ns": 1001083.29207, "command_buffers": 2 } ], "latency": [ { "compute_ns": 16500, "compute_span_ns": 16500, "calibration_cpu_ns": 417000, "calibration_gpu_ticks": 417000, "compute_passes": 1, "command_buffer_ns": 16500.0092238, "command_buffers": 2 }, { "compute_ns": 19083, "compute_span_ns": 19083, "calibration_cpu_ns": 416791, "calibration_gpu_ticks": 416791, "compute_passes": 1, "command_buffer_ns": 19041.6467376, "command_buffers": 2 }, { "compute_ns": 18875, "compute_span_ns": 18875, "calibration_cpu_ns": 423334, "calibration_gpu_ticks": 423334, "compute_passes": 1, "command_buffer_ns": 19000.0282601, "command_buffers": 2 }, { "compute_ns": 18875, "compute_span_ns": 18875, "calibration_cpu_ns": 437333, "calibration_gpu_ticks": 437333, "compute_passes": 1, "command_buffer_ns": 18958.2933672, "command_buffers": 2 }, { "compute_ns": 19125, "compute_span_ns": 19125, "calibration_cpu_ns": 459458, "calibration_gpu_ticks": 459458, "compute_passes": 1, "command_buffer_ns": 19208.3534785, "command_buffers": 2 }, { "compute_ns": 20667, "compute_span_ns": 20667, "calibration_cpu_ns": 617792, "calibration_gpu_ticks": 617792, "compute_passes": 1, "command_buffer_ns": 20666.688215, "command_buffers": 2 }, { "compute_ns": 19041, "compute_span_ns": 19041, "calibration_cpu_ns": 484667, "calibration_gpu_ticks": 484667, "compute_passes": 1, "command_buffer_ns": 19083.3234228, "command_buffers": 2 }, { "compute_ns": 19041, "compute_span_ns": 19041, "calibration_cpu_ns": 456125, "calibration_gpu_ticks": 456125, "compute_passes": 1, "command_buffer_ns": 18999.9700524, "command_buffers": 2 }, { "compute_ns": 20542, "compute_span_ns": 20542, "calibration_cpu_ns": 672959, "calibration_gpu_ticks": 672959, "compute_passes": 1, "command_buffer_ns": 20583.3348446, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 994999.951217, "command_buffers": 2 }, { "command_buffer_ns": 918666.657526, "command_buffers": 2 }, { "command_buffer_ns": 925708.271097, "command_buffers": 2 }, { "command_buffer_ns": 948125.030845, "command_buffers": 2 }, { "command_buffer_ns": 962541.67147, "command_buffers": 2 }, { "command_buffer_ns": 919583.253562, "command_buffers": 2 }, { "command_buffer_ns": 1023916.70225, "command_buffers": 2 }, { "command_buffer_ns": 914416.683372, "command_buffers": 2 }, { "command_buffer_ns": 925499.945879, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 17208.2800418, "command_buffers": 2 }, { "command_buffer_ns": 19458.2971744, "command_buffers": 2 }, { "command_buffer_ns": 19208.3534785, "command_buffers": 2 }, { "command_buffer_ns": 19166.6767932, "command_buffers": 2 }, { "command_buffer_ns": 19083.3816305, "command_buffers": 2 }, { "command_buffer_ns": 19291.6486412, "command_buffers": 2 }, { "command_buffer_ns": 18958.3515748, "command_buffers": 2 }, { "command_buffer_ns": 19249.971956, "command_buffers": 2 }, { "command_buffer_ns": 19208.3534785, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.546874237765625, 14.354166523843752, 14.464191735890624, 14.814453606953125, 15.03971361671875, 14.36848833690625, 15.99869847265625, 14.2877606776875, 14.460936654359376 ], "command_buffer_throughput_us_p50": 14.464191735890624, "command_buffer_throughput_us_p90": 15.637239084743749, "command_buffer_latency_us": [ 17.208280041800002, 19.4582971744, 19.2083534785, 19.1666767932, 19.0833816305, 19.291648641200002, 18.958351574799998, 19.249971956, 19.2083534785 ], "command_buffer_latency_us_p50": 19.2083534785, "command_buffer_latency_us_p90": 19.324978347840002 }, "compute_throughput_us": [ 15.205078125, 14.39909375, 15.524734375, 14.845703125, 14.3515625, 14.708984375, 14.193359375, 15.279953125, 15.641265625 ], "compute_throughput_us_p50": 14.845703125, "compute_throughput_us_p90": 15.548040625, "command_buffer_throughput_us": [ 15.205728232109376, 14.40104188078125, 15.525390153921874, 14.847006241328124, 14.353515325640625, 14.710936739000001, 14.19466207153125, 15.281901141859375, 15.64192643859375 ], "command_buffer_throughput_us_p50": 14.847006241328124, "command_buffer_throughput_us_p90": 15.54869741085625, "compute_latency_us": [ 16.5, 19.083, 18.875, 18.875, 19.125, 20.667, 19.041, 19.041, 20.542 ], "compute_latency_us_p50": 19.041, "compute_latency_us_p90": 20.567, "command_buffer_latency_us": [ 16.5000092238, 19.0416467376, 19.0000282601, 18.9582933672, 19.2083534785, 20.666688214999997, 19.0833234228, 18.9999700524, 20.583334844600003 ], "command_buffer_latency_us_p50": 19.0416467376, "command_buffer_latency_us_p90": 20.600005518680003, "counter_control_throughput_ratio": 1.003265627221172, "counter_control_latency_ratio": 0.9891544468591137 }, "correctness": { "max_abs_error": 1.6175184480715643e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 15.6171527697, "throughput_us_p90": 15.7494559767, "latency_us_p50": 280.167, "latency_us_p90": 325.4004 }, "slowdown": 0.7685104116497847, "device_compute_slowdown": 0.797363675044038, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-891z74qm/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "c9fd11968a5edbf77e894438f42e4bd465ad146629716a6f08eacac9ab82063f", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.600625, "capture_ms": 0.06775, "compile_ms": 52.604917, "allocation_upload_ms": 6.212416, "cold_call_ms": 7.81725, "warmup_ms": 200.188458, "download_ms": 3.879541, "repetitions": 443, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 65.9753589165, 65.3704853273, 66.6272573363, 66.5381873589, 66.600261851, 66.2517878104, 66.1798329571, 65.6318668172, 65.6719322799 ], "latency_us": [ 313.083, 287.625, 278, 274.041, 281.417, 268.083, 268.708, 271.084, 302.5 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 96, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4175500, "compute_span_ns": 4175500, "calibration_cpu_ns": 4668667, "calibration_gpu_ticks": 4668667, "compute_passes": 1, "command_buffer_ns": 4175625.04299, "command_buffers": 2 }, { "compute_ns": 4154625, "compute_span_ns": 4154625, "calibration_cpu_ns": 4659042, "calibration_gpu_ticks": 4659042, "compute_passes": 1, "command_buffer_ns": 4154666.67619, "command_buffers": 2 }, { "compute_ns": 4236709, "compute_span_ns": 4236709, "calibration_cpu_ns": 4715209, "calibration_gpu_ticks": 4715209, "compute_passes": 1, "command_buffer_ns": 4236875.04372, "command_buffers": 2 }, { "compute_ns": 4169875, "compute_span_ns": 4169875, "calibration_cpu_ns": 4741167, "calibration_gpu_ticks": 4741167, "compute_passes": 1, "command_buffer_ns": 4169999.97105, "command_buffers": 2 }, { "compute_ns": 4278083, "compute_span_ns": 4278083, "calibration_cpu_ns": 4811167, "calibration_gpu_ticks": 4811167, "compute_passes": 1, "command_buffer_ns": 4278166.68525, "command_buffers": 2 }, { "compute_ns": 4182833, "compute_span_ns": 4182833, "calibration_cpu_ns": 4611417, "calibration_gpu_ticks": 4611417, "compute_passes": 1, "command_buffer_ns": 4182958.39336, "command_buffers": 2 }, { "compute_ns": 4199000, "compute_span_ns": 4199000, "calibration_cpu_ns": 4631875, "calibration_gpu_ticks": 4631875, "compute_passes": 1, "command_buffer_ns": 4199041.69394, "command_buffers": 2 }, { "compute_ns": 4192208, "compute_span_ns": 4192208, "calibration_cpu_ns": 4699458, "calibration_gpu_ticks": 4699458, "compute_passes": 1, "command_buffer_ns": 4192208.34738, "command_buffers": 2 }, { "compute_ns": 4200417, "compute_span_ns": 4200417, "calibration_cpu_ns": 4880084, "calibration_gpu_ticks": 4880084, "compute_passes": 1, "command_buffer_ns": 4200541.70536, "command_buffers": 2 } ], "latency": [ { "compute_ns": 78458, "compute_span_ns": 78458, "calibration_cpu_ns": 566583, "calibration_gpu_ticks": 566583, "compute_passes": 1, "command_buffer_ns": 78500.0156611, "command_buffers": 2 }, { "compute_ns": 65250, "compute_span_ns": 65250, "calibration_cpu_ns": 465708, "calibration_gpu_ticks": 465708, "compute_passes": 1, "command_buffer_ns": 65250.0311844, "command_buffers": 2 }, { "compute_ns": 66416, "compute_span_ns": 66416, "calibration_cpu_ns": 439292, "calibration_gpu_ticks": 439292, "compute_passes": 1, "command_buffer_ns": 66375.0106469, "command_buffers": 2 }, { "compute_ns": 66459, "compute_span_ns": 66459, "calibration_cpu_ns": 424750, "calibration_gpu_ticks": 424750, "compute_passes": 1, "command_buffer_ns": 66541.6009724, "command_buffers": 2 }, { "compute_ns": 77875, "compute_span_ns": 77875, "calibration_cpu_ns": 527291, "calibration_gpu_ticks": 527291, "compute_passes": 1, "command_buffer_ns": 77874.9817982, "command_buffers": 2 }, { "compute_ns": 74834, "compute_span_ns": 74834, "calibration_cpu_ns": 582125, "calibration_gpu_ticks": 582125, "compute_passes": 1, "command_buffer_ns": 74958.3705328, "command_buffers": 2 }, { "compute_ns": 68333, "compute_span_ns": 68333, "calibration_cpu_ns": 487958, "calibration_gpu_ticks": 487958, "compute_passes": 1, "command_buffer_ns": 68416.7025611, "command_buffers": 2 }, { "compute_ns": 67542, "compute_span_ns": 67542, "calibration_cpu_ns": 495917, "calibration_gpu_ticks": 495917, "compute_passes": 1, "command_buffer_ns": 67583.3434798, "command_buffers": 2 }, { "compute_ns": 66875, "compute_span_ns": 66875, "calibration_cpu_ns": 465166, "calibration_gpu_ticks": 465166, "compute_passes": 1, "command_buffer_ns": 66875.0144541, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4252958.3443, "command_buffers": 2 }, { "command_buffer_ns": 4218000.04551, "command_buffers": 2 }, { "command_buffer_ns": 4286874.95878, "command_buffers": 2 }, { "command_buffer_ns": 4188125.02176, "command_buffers": 2 }, { "command_buffer_ns": 4297458.33389, "command_buffers": 2 }, { "command_buffer_ns": 4172833.28716, "command_buffers": 2 }, { "command_buffer_ns": 4235791.62452, "command_buffers": 2 }, { "command_buffer_ns": 4217916.69214, "command_buffers": 2 }, { "command_buffer_ns": 4190416.65737, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 75374.962762, "command_buffers": 2 }, { "command_buffer_ns": 65208.2962915, "command_buffers": 2 }, { "command_buffer_ns": 69374.9752827, "command_buffers": 2 }, { "command_buffer_ns": 66208.3621137, "command_buffers": 2 }, { "command_buffer_ns": 76833.2974985, "command_buffers": 2 }, { "command_buffer_ns": 67666.6386425, "command_buffers": 2 }, { "command_buffer_ns": 67083.3396725, "command_buffers": 2 }, { "command_buffer_ns": 77166.6527726, "command_buffers": 2 }, { "command_buffer_ns": 80375.0008345, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 66.4524741296875, 65.90625071109375, 66.9824212309375, 65.439453465, 67.14778646703125, 65.200520111875, 66.184244133125, 65.9049483146875, 65.47526027140626 ], "command_buffer_throughput_us_p50": 65.90625071109375, "command_buffer_throughput_us_p90": 67.01549427815625, "command_buffer_latency_us": [ 75.374962762, 65.2082962915, 69.3749752827, 66.2083621137, 76.8332974985, 67.66663864249999, 67.0833396725, 77.1666527726, 80.3750008345 ], "command_buffer_latency_us_p50": 69.3749752827, "command_buffer_latency_us_p90": 77.80832238498 }, "compute_throughput_us": [ 65.2421875, 64.916015625, 66.198578125, 65.154296875, 66.845046875, 65.356765625, 65.609375, 65.50325, 65.631515625 ], "compute_throughput_us_p50": 65.50325, "compute_throughput_us_p90": 66.327871875, "command_buffer_throughput_us": [ 65.24414129671875, 64.91666681546874, 66.20117255812501, 65.15624954765624, 66.84635445703125, 65.35872489625, 65.61002646781249, 65.5032554278125, 65.63346414625 ], "command_buffer_throughput_us_p50": 65.5032554278125, "command_buffer_throughput_us_p90": 66.33020893790626, "compute_latency_us": [ 78.458, 65.25, 66.416, 66.459, 77.875, 74.834, 68.333, 67.542, 66.875 ], "compute_latency_us_p50": 67.542, "compute_latency_us_p90": 77.9916, "command_buffer_latency_us": [ 78.5000156611, 65.2500311844, 66.37501064690001, 66.5416009724, 77.8749817982, 74.9583705328, 68.4167025611, 67.58334347979999, 66.87501445410001 ], "command_buffer_latency_us_p50": 67.58334347979999, "command_buffer_latency_us_p90": 77.99998857077999, "counter_control_throughput_ratio": 0.9939049662104739, "counter_control_latency_ratio": 1.0050331838465922 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 66.1798329571, "throughput_us_p90": 66.60566094806, "latency_us_p50": 278.0, "latency_us_p90": 304.6166 }, "torch": { "cold_call_ms": 9.732625, "warmup_ms": 200.142709, "repetitions": 291, "throughput_us": [ 82.07087628865979, 80.68800343642611, 81.40864948453608, 80.88860137457044, 83.66752577319588, 81.1877147766323, 81.05970790378007, 80.93427835051547, 80.96535051546392 ], "latency_us": [ 438.625, 339.83299999999997, 301.417, 404.75, 315.416, 313.625, 295.04200000000003, 335.334, 321.79200000000003 ], "throughput_us_p50": 81.05970790378007, "throughput_us_p90": 82.39020618556701, "latency_us_p50": 321.79200000000003, "latency_us_p90": 411.525, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5029666.70319438, "command_buffers": 1 }, { "command_buffer_ns": 5069500.009994954, "command_buffers": 1 }, { "command_buffer_ns": 5097749.992273748, "command_buffers": 1 }, { "command_buffer_ns": 5099833.360873163, "command_buffers": 1 }, { "command_buffer_ns": 5202125.001233071, "command_buffers": 1 }, { "command_buffer_ns": 5056291.643995792, "command_buffers": 1 }, { "command_buffer_ns": 4915374.971460551, "command_buffers": 1 }, { "command_buffer_ns": 4976583.3537094295, "command_buffers": 1 }, { "command_buffer_ns": 5059083.3416208625, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 98499.99332800508, "command_buffers": 1 }, { "command_buffer_ns": 80499.97268244624, "command_buffers": 1 }, { "command_buffer_ns": 80916.68132692575, "command_buffers": 1 }, { "command_buffer_ns": 97499.9857135117, "command_buffers": 1 }, { "command_buffer_ns": 80124.99893084168, "command_buffers": 1 }, { "command_buffer_ns": 80499.97268244624, "command_buffers": 1 }, { "command_buffer_ns": 80000.02708286047, "command_buffers": 1 }, { "command_buffer_ns": 96791.71489551663, "command_buffers": 1 }, { "command_buffer_ns": 81041.65317490697, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 78.58854223741218, 79.21093765617115, 79.65234362927731, 79.68489626364317, 81.28320314426674, 79.00455693743424, 76.80273392907111, 77.75911490170984, 79.04817721282598 ], "command_buffer_throughput_us_p50": 79.04817721282598, "command_buffer_throughput_us_p90": 80.00455763976788, "command_buffer_latency_us": [ 98.49999332800508, 80.49997268244624, 80.91668132692575, 97.4999857135117, 80.12499893084168, 80.49997268244624, 80.00002708286047, 96.79171489551663, 81.04165317490697 ], "command_buffer_latency_us_p50": 80.91668132692575, "command_buffer_latency_us_p90": 97.69998723641038 }, "throughput": [ { "compute_ns": 5031459.0, "compute_span_ns": 5031459.0, "command_buffer_ns": 5031499.953474849, "calibration_cpu_ns": 5672250.0, "calibration_gpu_ticks": 5672250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5102334.0, "compute_span_ns": 5102334.0, "command_buffer_ns": 5102374.998386949, "calibration_cpu_ns": 5757250.0, "calibration_gpu_ticks": 5757250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5055833.0, "compute_span_ns": 5055833.0, "command_buffer_ns": 5055791.640188545, "calibration_cpu_ns": 5735834.0, "calibration_gpu_ticks": 5735834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5098000.0, "compute_span_ns": 5098000.0, "command_buffer_ns": 5097999.994177371, "calibration_cpu_ns": 5796333.0, "calibration_gpu_ticks": 5796333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5138750.0, "compute_span_ns": 5138750.0, "command_buffer_ns": 5138749.955222011, "calibration_cpu_ns": 5868167.0, "calibration_gpu_ticks": 5868167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5108458.0, "compute_span_ns": 5108458.0, "command_buffer_ns": 5108416.662551463, "calibration_cpu_ns": 5848500.0, "calibration_gpu_ticks": 5848500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4944792.0, "compute_span_ns": 4944792.0, "command_buffer_ns": 4944791.668094695, "calibration_cpu_ns": 5781458.0, "calibration_gpu_ticks": 5781458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4949459.0, "compute_span_ns": 4949459.0, "command_buffer_ns": 4949500.027578324, "calibration_cpu_ns": 5756875.0, "calibration_gpu_ticks": 5756875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5026250.0, "compute_span_ns": 5026250.0, "command_buffer_ns": 5026250.029914081, "calibration_cpu_ns": 5826167.0, "calibration_gpu_ticks": 5826167.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 96458.0, "compute_span_ns": 96458.0, "command_buffer_ns": 96458.30141380429, "calibration_cpu_ns": 600167.0, "calibration_gpu_ticks": 600167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 79709.0, "compute_span_ns": 79709.0, "command_buffer_ns": 79749.96697157621, "calibration_cpu_ns": 527750.0, "calibration_gpu_ticks": 527750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80375.0, "compute_span_ns": 80375.0, "command_buffer_ns": 80375.00083446503, "calibration_cpu_ns": 573584.0, "calibration_gpu_ticks": 573584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80708.0, "compute_span_ns": 80708.0, "command_buffer_ns": 80666.67942330241, "calibration_cpu_ns": 578333.0, "calibration_gpu_ticks": 578333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80375.0, "compute_span_ns": 80375.0, "command_buffer_ns": 80375.00083446503, "calibration_cpu_ns": 585333.0, "calibration_gpu_ticks": 585333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 81375.0, "compute_span_ns": 81375.0, "command_buffer_ns": 81374.95024129748, "calibration_cpu_ns": 590500.0, "calibration_gpu_ticks": 590500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80208.0, "compute_span_ns": 80208.0, "command_buffer_ns": 80166.67561605573, "calibration_cpu_ns": 575584.0, "calibration_gpu_ticks": 575584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 99625.0, "compute_span_ns": 99625.0, "command_buffer_ns": 99625.03099814057, "calibration_cpu_ns": 664792.0, "calibration_gpu_ticks": 664792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80042.0, "compute_span_ns": 80042.0, "command_buffer_ns": 80083.32224562764, "calibration_cpu_ns": 563875.0, "calibration_gpu_ticks": 563875.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 78.616546875, 79.72396875, 78.997390625, 79.65625, 80.29296875, 79.81965625, 77.262375, 77.335296875, 78.53515625 ], "compute_throughput_us_p50": 78.997390625, "compute_throughput_us_p90": 79.91431874999999, "command_buffer_throughput_us": [ 78.61718677304452, 79.72460934979608, 78.99674437794602, 79.65624990902143, 80.29296805034392, 79.8190103523666, 77.2623698139796, 77.33593793091131, 78.53515671740752 ], "command_buffer_throughput_us_p50": 78.99674437794602, "command_buffer_throughput_us_p90": 79.91380189196207, "compute_latency_us": [ 96.458, 79.709, 80.375, 80.708, 80.375, 81.375, 80.208, 99.625, 80.042 ], "compute_latency_us_p50": 80.375, "compute_latency_us_p90": 97.0914, "command_buffer_latency_us": [ 96.45830141380429, 79.74996697157621, 80.37500083446503, 80.66667942330241, 80.37500083446503, 81.37495024129748, 80.16667561605573, 99.62503099814057, 80.08332224562764 ], "command_buffer_latency_us_p50": 80.37500083446503, "command_buffer_latency_us_p90": 97.09164733067155, "counter_control_throughput_ratio": 0.9996405045878837, "counter_control_latency_ratio": 0.9933057005851218 }, "download_ms": 0.809208, "allocation_upload_ms": 15.88125, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.8164331536409821, "device_compute_slowdown": 0.8291824512399836, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "rmsnorm", "m": 768, "n": 6144, "k": 1 }, "name": "rmsnorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.4265, "warmup_ms": 200.446792, "repetitions": 356, "throughput_us": [ 81.57560955056181, 81.70446910112359, 81.76650280898876, 81.83040730337079, 81.87371067415731, 81.9170168539326, 81.8560393258427, 82.0886011235955, 81.68094662921348 ], "latency_us": [ 336.5, 312.042, 327.33299999999997, 325.083, 370.70799999999997, 263.5, 462.083, 377.70799999999997, 341.75 ], "throughput_us_p50": 81.83040730337079, "throughput_us_p90": 81.95133370786517, "latency_us_p50": 336.5, "latency_us_p90": 394.58299999999997, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 5048999.970313162, "command_buffers": 1 }, { "command_buffer_ns": 5083750.002086163, "command_buffers": 1 }, { "command_buffer_ns": 5074916.640296578, "command_buffers": 1 }, { "command_buffer_ns": 5114208.383020014, "command_buffers": 1 }, { "command_buffer_ns": 5114166.648127139, "command_buffers": 1 }, { "command_buffer_ns": 5111625.010613352, "command_buffers": 1 }, { "command_buffer_ns": 5103250.034153461, "command_buffers": 1 }, { "command_buffer_ns": 5100124.981254339, "command_buffers": 1 }, { "command_buffer_ns": 5057791.655417532, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 94458.34439247847, "command_buffers": 1 }, { "command_buffer_ns": 81708.30551534891, "command_buffers": 1 }, { "command_buffer_ns": 81125.00654533505, "command_buffers": 1 }, { "command_buffer_ns": 79874.99702721834, "command_buffers": 1 }, { "command_buffer_ns": 81083.32986012101, "command_buffers": 1 }, { "command_buffer_ns": 80124.99893084168, "command_buffers": 1 }, { "command_buffer_ns": 81875.01225620508, "command_buffers": 1 }, { "command_buffer_ns": 114374.96868893504, "command_buffers": 1 }, { "command_buffer_ns": 128749.99083578587, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 78.89062453614315, 79.43359378259629, 79.29557250463404, 79.90950598468771, 79.90885387698654, 79.86914079083363, 79.73828178364784, 79.68945283209905, 79.02799461589893 ], "command_buffer_throughput_us_p50": 79.68945283209905, "command_buffer_throughput_us_p90": 79.90898429852678, "command_buffer_latency_us": [ 94.45834439247847, 81.70830551534891, 81.12500654533505, 79.87499702721834, 81.08332986012101, 80.12499893084168, 81.87501225620508, 114.37496868893504, 128.74999083578587 ], "command_buffer_latency_us_p50": 81.70830551534891, "command_buffer_latency_us_p90": 117.2499731183052 }, "throughput": [ { "compute_ns": 5076959.0, "compute_span_ns": 5076959.0, "command_buffer_ns": 5077000.008895993, "calibration_cpu_ns": 5851500.0, "calibration_gpu_ticks": 5851500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5118625.0, "compute_span_ns": 5118625.0, "command_buffer_ns": 5118625.005707145, "calibration_cpu_ns": 5756666.0, "calibration_gpu_ticks": 5756666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5099334.0, "compute_span_ns": 5099334.0, "command_buffer_ns": 5099374.975543469, "calibration_cpu_ns": 5797167.0, "calibration_gpu_ticks": 5797167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5069292.0, "compute_span_ns": 5069292.0, "command_buffer_ns": 5069333.361461759, "calibration_cpu_ns": 5734375.0, "calibration_gpu_ticks": 5734375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5072458.0, "compute_span_ns": 5072458.0, "command_buffer_ns": 5072416.679468006, "calibration_cpu_ns": 5858208.0, "calibration_gpu_ticks": 5858208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5077833.0, "compute_span_ns": 5077833.0, "command_buffer_ns": 5077833.367977291, "calibration_cpu_ns": 5815250.0, "calibration_gpu_ticks": 5815250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5053084.0, "compute_span_ns": 5053084.0, "command_buffer_ns": 5053124.972619116, "calibration_cpu_ns": 5840875.0, "calibration_gpu_ticks": 5840875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5095292.0, "compute_span_ns": 5095292.0, "command_buffer_ns": 5095291.649922729, "calibration_cpu_ns": 5921583.0, "calibration_gpu_ticks": 5921583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 5097833.0, "compute_span_ns": 5097833.0, "command_buffer_ns": 5097791.668958962, "calibration_cpu_ns": 6213334.0, "calibration_gpu_ticks": 6213334.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 81042.0, "compute_span_ns": 81042.0, "command_buffer_ns": 81083.32986012101, "calibration_cpu_ns": 671750.0, "calibration_gpu_ticks": 671750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80333.0, "compute_span_ns": 80333.0, "command_buffer_ns": 80291.70567169785, "calibration_cpu_ns": 557125.0, "calibration_gpu_ticks": 557125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 100042.0, "compute_span_ns": 100042.0, "command_buffer_ns": 100041.62322729826, "calibration_cpu_ns": 701125.0, "calibration_gpu_ticks": 701125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 80542.0, "compute_span_ns": 80542.0, "command_buffer_ns": 80583.32605287433, "calibration_cpu_ns": 672125.0, "calibration_gpu_ticks": 672125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82750.0, "compute_span_ns": 82750.0, "command_buffer_ns": 82749.98981505632, "calibration_cpu_ns": 1736625.0, "calibration_gpu_ticks": 1736625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 99667.0, "compute_span_ns": 99667.0, "command_buffer_ns": 99666.6494756937, "calibration_cpu_ns": 1541167.0, "calibration_gpu_ticks": 1541167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 98166.0, "compute_span_ns": 98166.0, "command_buffer_ns": 98125.01957640052, "calibration_cpu_ns": 1393583.0, "calibration_gpu_ticks": 1393583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 97125.0, "compute_span_ns": 97125.0, "command_buffer_ns": 97125.01196190715, "calibration_cpu_ns": 880041.0, "calibration_gpu_ticks": 880041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 101416.0, "compute_span_ns": 101416.0, "command_buffer_ns": 101375.04432350397, "calibration_cpu_ns": 912417.0, "calibration_gpu_ticks": 912417.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 79.327484375, 79.978515625, 79.67709375, 79.2076875, 79.25715625, 79.341140625, 78.9544375, 79.6139375, 79.653640625 ], "compute_throughput_us_p50": 79.341140625, "compute_throughput_us_p90": 79.73737812499999, "command_buffer_throughput_us": [ 79.3281251389999, 79.97851571417414, 79.6777339928667, 79.20833377283998, 79.25651061668759, 79.34114637464518, 78.95507769717369, 79.61393203004263, 79.65299482748378 ], "command_buffer_throughput_us_p50": 79.34114637464518, "command_buffer_throughput_us_p90": 79.73789033712819, "compute_latency_us": [ 81.042, 80.333, 100.042, 80.542, 82.75, 99.667, 98.166, 97.125, 101.416 ], "compute_latency_us_p50": 97.125, "compute_latency_us_p90": 100.3168, "command_buffer_latency_us": [ 81.08332986012101, 80.29170567169785, 100.04162322729826, 80.58332605287433, 82.74998981505632, 99.6666494756937, 98.12501957640052, 97.12501196190715, 101.37504432350397 ], "command_buffer_latency_us_p50": 97.12501196190715, "command_buffer_latency_us_p90": 100.3083074465394, "counter_control_throughput_ratio": 0.9990523111983773, "counter_control_latency_ratio": 1.0088679693523446 }, "download_ms": 1.468958, "allocation_upload_ms": 1.402417, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.9755493041451189e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-6bg4vbyf/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "232522da57db77c64b377ec2facdc2e1df33cd57064a4297b89d6582fd4ad63b", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.033, "capture_ms": 0.061, "compile_ms": 52.230208, "allocation_upload_ms": 6.765084, "cold_call_ms": 5.148917, "warmup_ms": 200.736958, "download_ms": 4.271375, "repetitions": 482, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 60.7901099585, 60.1186037344, 60.0372572614, 59.9086286307, 59.8719751037, 60.0632780083, 60.0026784232, 60.174186722, 60.2247593361 ], "latency_us": [ 362.958, 321.042, 362, 305.666, 293.584, 304.208, 296.542, 309.083, 278.667 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 72, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 24576, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 96, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 1, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 104, "normalized_kernel_cost": 1248, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3895625, "compute_span_ns": 3895625, "calibration_cpu_ns": 4394167, "calibration_gpu_ticks": 4394167, "compute_passes": 1, "command_buffer_ns": 3895625.00641, "command_buffers": 2 }, { "compute_ns": 3833042, "compute_span_ns": 3833042, "calibration_cpu_ns": 4339583, "calibration_gpu_ticks": 4339583, "compute_passes": 1, "command_buffer_ns": 3833083.31948, "command_buffers": 2 }, { "compute_ns": 3924625, "compute_span_ns": 3924625, "calibration_cpu_ns": 4418459, "calibration_gpu_ticks": 4418459, "compute_passes": 1, "command_buffer_ns": 3924708.34777, "command_buffers": 2 }, { "compute_ns": 3938958, "compute_span_ns": 3938958, "calibration_cpu_ns": 4386042, "calibration_gpu_ticks": 4386042, "compute_passes": 1, "command_buffer_ns": 3938999.95834, "command_buffers": 2 }, { "compute_ns": 3923959, "compute_span_ns": 3923959, "calibration_cpu_ns": 4382834, "calibration_gpu_ticks": 4382834, "compute_passes": 1, "command_buffer_ns": 3924041.69543, "command_buffers": 2 }, { "compute_ns": 3948041, "compute_span_ns": 3948041, "calibration_cpu_ns": 4407667, "calibration_gpu_ticks": 4407667, "compute_passes": 1, "command_buffer_ns": 3948041.70355, "command_buffers": 2 }, { "compute_ns": 3894250, "compute_span_ns": 3894250, "calibration_cpu_ns": 4371500, "calibration_gpu_ticks": 4371500, "compute_passes": 1, "command_buffer_ns": 3894374.93868, "command_buffers": 2 }, { "compute_ns": 3902500, "compute_span_ns": 3902500, "calibration_cpu_ns": 4606000, "calibration_gpu_ticks": 4606000, "compute_passes": 1, "command_buffer_ns": 3902541.64813, "command_buffers": 2 }, { "compute_ns": 3898292, "compute_span_ns": 3898292, "calibration_cpu_ns": 4393833, "calibration_gpu_ticks": 4393833, "compute_passes": 1, "command_buffer_ns": 3898458.32251, "command_buffers": 2 } ], "latency": [ { "compute_ns": 60875, "compute_span_ns": 60875, "calibration_cpu_ns": 507375, "calibration_gpu_ticks": 507375, "compute_passes": 1, "command_buffer_ns": 60958.2639299, "command_buffers": 2 }, { "compute_ns": 78167, "compute_span_ns": 78167, "calibration_cpu_ns": 614875, "calibration_gpu_ticks": 614875, "compute_passes": 1, "command_buffer_ns": 78208.3370723, "command_buffers": 2 }, { "compute_ns": 65167, "compute_span_ns": 65167, "calibration_cpu_ns": 537958, "calibration_gpu_ticks": 537958, "compute_passes": 1, "command_buffer_ns": 65208.2962915, "command_buffers": 2 }, { "compute_ns": 60750, "compute_span_ns": 60750, "calibration_cpu_ns": 566000, "calibration_gpu_ticks": 566000, "compute_passes": 1, "command_buffer_ns": 60791.6736044, "command_buffers": 2 }, { "compute_ns": 75417, "compute_span_ns": 75417, "calibration_cpu_ns": 539833, "calibration_gpu_ticks": 539833, "compute_passes": 1, "command_buffer_ns": 75458.3161324, "command_buffers": 2 }, { "compute_ns": 70250, "compute_span_ns": 70250, "calibration_cpu_ns": 514416, "calibration_gpu_ticks": 514416, "compute_passes": 1, "command_buffer_ns": 70291.6295268, "command_buffers": 2 }, { "compute_ns": 61458, "compute_span_ns": 61458, "calibration_cpu_ns": 491292, "calibration_gpu_ticks": 491292, "compute_passes": 1, "command_buffer_ns": 61458.3259448, "command_buffers": 2 }, { "compute_ns": 75667, "compute_span_ns": 75667, "calibration_cpu_ns": 568125, "calibration_gpu_ticks": 568125, "compute_passes": 1, "command_buffer_ns": 75749.9947213, "command_buffers": 2 }, { "compute_ns": 61583, "compute_span_ns": 61583, "calibration_cpu_ns": 504041, "calibration_gpu_ticks": 504041, "compute_passes": 1, "command_buffer_ns": 61583.3560005, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3923458.39646, "command_buffers": 2 }, { "command_buffer_ns": 3859874.90881, "command_buffers": 2 }, { "command_buffer_ns": 3892125.03796, "command_buffers": 2 }, { "command_buffer_ns": 3841958.29974, "command_buffers": 2 }, { "command_buffer_ns": 3973083.33777, "command_buffers": 2 }, { "command_buffer_ns": 4015666.66551, "command_buffers": 2 }, { "command_buffer_ns": 3914791.6832, "command_buffers": 2 }, { "command_buffer_ns": 3846708.33591, "command_buffers": 2 }, { "command_buffer_ns": 3870000.01501, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 71208.3419785, "command_buffers": 2 }, { "command_buffer_ns": 73833.3328627, "command_buffers": 2 }, { "command_buffer_ns": 72708.3534002, "command_buffers": 2 }, { "command_buffer_ns": 75249.990914, "command_buffers": 2 }, { "command_buffer_ns": 75999.9966249, "command_buffers": 2 }, { "command_buffer_ns": 62083.3598077, "command_buffers": 2 }, { "command_buffer_ns": 62083.3598077, "command_buffers": 2 }, { "command_buffer_ns": 73500.0357963, "command_buffers": 2 }, { "command_buffer_ns": 61416.6492596, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 61.3040374446875, 60.31054545015625, 60.814453718124994, 60.030598433437504, 62.07942715265625, 62.744791648593754, 61.16862005, 60.104817748593746, 60.46875023453126 ], "command_buffer_throughput_us_p50": 60.814453718124994, "command_buffer_throughput_us_p90": 62.212500051843755, "command_buffer_latency_us": [ 71.2083419785, 73.8333328627, 72.7083534002, 75.249990914, 75.99999662489999, 62.0833598077, 62.0833598077, 73.5000357963, 61.4166492596 ], "command_buffer_latency_us_p50": 72.7083534002, "command_buffer_latency_us_p90": 75.39999205618 }, "compute_throughput_us": [ 60.869140625, 59.89128125, 61.322265625, 61.54621875, 61.311859375, 61.688140625, 60.84765625, 60.9765625, 60.9108125 ], "compute_throughput_us_p50": 60.9765625, "compute_throughput_us_p90": 61.574603125, "command_buffer_throughput_us": [ 60.86914072515625, 59.891926866875004, 61.32356793390625, 61.5468743490625, 61.31315149109375, 61.68815161796876, 60.849608416875, 60.97721325203125, 60.91341128921875 ], "command_buffer_throughput_us_p50": 60.97721325203125, "command_buffer_throughput_us_p90": 61.57512980284375, "compute_latency_us": [ 60.875, 78.167, 65.167, 60.75, 75.417, 70.25, 61.458, 75.667, 61.583 ], "compute_latency_us_p50": 65.167, "compute_latency_us_p90": 76.167, "command_buffer_latency_us": [ 60.9582639299, 78.2083370723, 65.2082962915, 60.791673604399996, 75.4583161324, 70.2916295268, 61.4583259448, 75.7499947213, 61.583356000500004 ], "command_buffer_latency_us_p50": 65.2082962915, "command_buffer_latency_us_p90": 76.2416631915, "counter_control_throughput_ratio": 0.9947847175093335, "counter_control_latency_ratio": 0.9928726247821631 }, "correctness": { "max_abs_error": 1.4915225321487924e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 60.0632780083, "throughput_us_p90": 60.33782946058, "latency_us_p50": 305.666, "latency_us_p90": 362.1916 }, "slowdown": 0.733997055466518, "device_compute_slowdown": 0.7685364997234057, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-ug2g8zog/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "26111b6b600e52c50494ad3f783ef6fc3b02252d329d47ea39eb66749773d18e", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.699875, "capture_ms": 0.06925, "compile_ms": 52.188, "allocation_upload_ms": 5.665542, "cold_call_ms": 5.79275, "warmup_ms": 200.559125, "download_ms": 3.072125, "repetitions": 524, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 53.2163645038, 53.5580458015, 52.3951965649, 53.4526889313, 53.3884370229, 52.8742843511, 53.6303263359, 53.4328874046, 53.4165877863 ], "latency_us": [ 286.083, 264.708, 347.209, 261.25, 265.334, 277.833, 258.917, 259.208, 266.917 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 32, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 1280, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3353625, "compute_span_ns": 3353625, "calibration_cpu_ns": 3833417, "calibration_gpu_ticks": 3833417, "compute_passes": 1, "command_buffer_ns": 3353666.68878, "command_buffers": 2 }, { "compute_ns": 3426292, "compute_span_ns": 3426292, "calibration_cpu_ns": 3893583, "calibration_gpu_ticks": 3893583, "compute_passes": 1, "command_buffer_ns": 3426333.3655, "command_buffers": 2 }, { "compute_ns": 3312125, "compute_span_ns": 3312125, "calibration_cpu_ns": 3840666, "calibration_gpu_ticks": 3840666, "compute_passes": 1, "command_buffer_ns": 3312124.98713, "command_buffers": 2 }, { "compute_ns": 3366750, "compute_span_ns": 3366750, "calibration_cpu_ns": 3978750, "calibration_gpu_ticks": 3978750, "compute_passes": 1, "command_buffer_ns": 3366791.6432, "command_buffers": 2 }, { "compute_ns": 3387833, "compute_span_ns": 3387833, "calibration_cpu_ns": 3893542, "calibration_gpu_ticks": 3893542, "compute_passes": 1, "command_buffer_ns": 3387875.04006, "command_buffers": 2 }, { "compute_ns": 3352708, "compute_span_ns": 3352708, "calibration_cpu_ns": 3789458, "calibration_gpu_ticks": 3789458, "compute_passes": 1, "command_buffer_ns": 3352791.71122, "command_buffers": 2 }, { "compute_ns": 3404875, "compute_span_ns": 3404875, "calibration_cpu_ns": 3836958, "calibration_gpu_ticks": 3836958, "compute_passes": 1, "command_buffer_ns": 3404958.40646, "command_buffers": 2 }, { "compute_ns": 3367584, "compute_span_ns": 3367584, "calibration_cpu_ns": 3840291, "calibration_gpu_ticks": 3840291, "compute_passes": 1, "command_buffer_ns": 3367708.35565, "command_buffers": 2 }, { "compute_ns": 3391166, "compute_span_ns": 3391166, "calibration_cpu_ns": 3829542, "calibration_gpu_ticks": 3829542, "compute_passes": 1, "command_buffer_ns": 3391166.68329, "command_buffers": 2 } ], "latency": [ { "compute_ns": 54125, "compute_span_ns": 54125, "calibration_cpu_ns": 475834, "calibration_gpu_ticks": 475834, "compute_passes": 1, "command_buffer_ns": 54166.7104699, "command_buffers": 2 }, { "compute_ns": 64500, "compute_span_ns": 64500, "calibration_cpu_ns": 492500, "calibration_gpu_ticks": 492500, "compute_passes": 1, "command_buffer_ns": 64541.7021587, "command_buffers": 2 }, { "compute_ns": 53833, "compute_span_ns": 53833, "calibration_cpu_ns": 443167, "calibration_gpu_ticks": 443167, "compute_passes": 1, "command_buffer_ns": 53874.9736734, "command_buffers": 2 }, { "compute_ns": 54834, "compute_span_ns": 54834, "calibration_cpu_ns": 422833, "calibration_gpu_ticks": 422833, "compute_passes": 1, "command_buffer_ns": 54999.9531358, "command_buffers": 2 }, { "compute_ns": 58125, "compute_span_ns": 58125, "calibration_cpu_ns": 532875, "calibration_gpu_ticks": 532875, "compute_passes": 1, "command_buffer_ns": 58166.6245125, "command_buffers": 2 }, { "compute_ns": 64292, "compute_span_ns": 64292, "calibration_cpu_ns": 488417, "calibration_gpu_ticks": 488417, "compute_passes": 1, "command_buffer_ns": 64333.3769403, "command_buffers": 2 }, { "compute_ns": 57833, "compute_span_ns": 57833, "calibration_cpu_ns": 461458, "calibration_gpu_ticks": 461458, "compute_passes": 1, "command_buffer_ns": 57833.3274461, "command_buffers": 2 }, { "compute_ns": 54334, "compute_span_ns": 54334, "calibration_cpu_ns": 451041, "calibration_gpu_ticks": 451041, "compute_passes": 1, "command_buffer_ns": 54416.6541658, "command_buffers": 2 }, { "compute_ns": 53916, "compute_span_ns": 53916, "calibration_cpu_ns": 510166, "calibration_gpu_ticks": 510166, "compute_passes": 1, "command_buffer_ns": 53875.031881, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3479333.36161, "command_buffers": 2 }, { "command_buffer_ns": 3448875.03888, "command_buffers": 2 }, { "command_buffer_ns": 3331083.39692, "command_buffers": 2 }, { "command_buffer_ns": 3468916.63503, "command_buffers": 2 }, { "command_buffer_ns": 3350958.34453, "command_buffers": 2 }, { "command_buffer_ns": 3365875.04717, "command_buffers": 2 }, { "command_buffer_ns": 3455666.65055, "command_buffers": 2 }, { "command_buffer_ns": 3395666.65934, "command_buffers": 2 }, { "command_buffer_ns": 3374916.61776, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 54333.3590031, "command_buffers": 2 }, { "command_buffer_ns": 66333.3339617, "command_buffers": 2 }, { "command_buffer_ns": 54500.0075363, "command_buffers": 2 }, { "command_buffer_ns": 54374.9774806, "command_buffers": 2 }, { "command_buffer_ns": 54333.3590031, "command_buffers": 2 }, { "command_buffer_ns": 65708.3000988, "command_buffers": 2 }, { "command_buffer_ns": 57250.0284761, "command_buffers": 2 }, { "command_buffer_ns": 55000.0113435, "command_buffers": 2 }, { "command_buffer_ns": 56833.3198316, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 54.36458377515625, 53.8886724825, 52.048178076875004, 54.20182242234375, 52.35872413328125, 52.59179761203125, 53.994791414843746, 53.05729155218749, 52.7330721525 ], "command_buffer_throughput_us_p50": 53.05729155218749, "command_buffer_throughput_us_p90": 54.23437469290625, "command_buffer_latency_us": [ 54.3333590031, 66.3333339617, 54.5000075363, 54.3749774806, 54.3333590031, 65.7083000988, 57.2500284761, 55.0000113435, 56.8333198316 ], "command_buffer_latency_us_p50": 55.0000113435, "command_buffer_latency_us_p90": 65.83330687138 }, "compute_throughput_us": [ 52.400390625, 53.5358125, 51.751953125, 52.60546875, 52.934890625, 52.3860625, 53.201171875, 52.6185, 52.98696875 ], "compute_throughput_us_p50": 52.6185, "compute_throughput_us_p90": 53.2681, "command_buffer_throughput_us": [ 52.4010420121875, 53.536458835937495, 51.751952923906245, 52.606119424999996, 52.9355475009375, 52.3873704878125, 53.202475100937505, 52.620443057031245, 52.98697942640625 ], "command_buffer_throughput_us_p50": 52.620443057031245, "command_buffer_throughput_us_p90": 53.2692718479375, "compute_latency_us": [ 54.125, 64.5, 53.833, 54.834, 58.125, 64.292, 57.833, 54.334, 53.916 ], "compute_latency_us_p50": 54.834, "compute_latency_us_p90": 64.3336, "command_buffer_latency_us": [ 54.166710469899996, 64.5417021587, 53.87497367340001, 54.999953135800006, 58.1666245125, 64.3333769403, 57.83332744609999, 54.4166541658, 53.875031881000005 ], "command_buffer_latency_us_p50": 54.999953135800006, "command_buffer_latency_us_p90": 64.37504198398, "counter_control_throughput_ratio": 0.9934640504147345, "counter_control_latency_ratio": 0.9893935080475407 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 53.4165877863, "throughput_us_p90": 53.57250190838, "latency_us_p50": 265.334, "latency_us_p90": 298.30820000000006 }, "torch": { "cold_call_ms": 7.380666, "warmup_ms": 200.218166, "repetitions": 406, "throughput_us": [ 73.30326354679804, 74.56999261083745, 72.35601231527095, 71.3527315270936, 72.05726600985221, 71.25082266009852, 71.2481527093596, 71.99404926108375, 71.02575862068966 ], "latency_us": [ 456.166, 369.334, 309.70799999999997, 317.541, 320.083, 453.125, 367.042, 331.45799999999997, 314.458 ], "throughput_us_p50": 71.99404926108375, "throughput_us_p90": 73.55660935960591, "latency_us_p50": 331.45799999999997, "latency_us_p90": 453.7332, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4405875.049997121, "command_buffers": 1 }, { "command_buffer_ns": 4391374.997794628, "command_buffers": 1 }, { "command_buffer_ns": 4398416.669573635, "command_buffers": 1 }, { "command_buffer_ns": 4506708.355620503, "command_buffers": 1 }, { "command_buffer_ns": 4379583.289846778, "command_buffers": 1 }, { "command_buffer_ns": 4515541.659202427, "command_buffers": 1 }, { "command_buffer_ns": 4476833.331864327, "command_buffers": 1 }, { "command_buffer_ns": 4424791.666679084, "command_buffers": 1 }, { "command_buffer_ns": 4383624.996989965, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 86625.04842504859, "command_buffers": 1 }, { "command_buffer_ns": 69999.95093792677, "command_buffers": 1 }, { "command_buffer_ns": 71249.96045604348, "command_buffers": 1 }, { "command_buffer_ns": 84125.02938881516, "command_buffers": 1 }, { "command_buffer_ns": 70666.71969369054, "command_buffers": 1 }, { "command_buffer_ns": 70791.63333401084, "command_buffers": 1 }, { "command_buffer_ns": 73333.3290554583, "command_buffers": 1 }, { "command_buffer_ns": 85458.33406969905, "command_buffers": 1 }, { "command_buffer_ns": 87249.96587261558, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 68.84179765620502, 68.61523434054106, 68.72526046208804, 70.41731805657037, 68.4309889038559, 70.55533842503792, 69.95052081038011, 69.13736979186069, 68.49414057796821 ], "command_buffer_throughput_us_p50": 68.84179765620502, "command_buffer_throughput_us_p90": 70.44492213026388, "command_buffer_latency_us": [ 86.62504842504859, 69.99995093792677, 71.24996045604348, 84.12502938881516, 70.66671969369054, 70.79163333401084, 73.33332905545831, 85.45833406969905, 87.24996587261558 ], "command_buffer_latency_us_p50": 73.33332905545831, "command_buffer_latency_us_p90": 86.75003191456199 }, "throughput": [ { "compute_ns": 4493125.0, "compute_span_ns": 4493125.0, "command_buffer_ns": 4493125.015869737, "calibration_cpu_ns": 5212041.0, "calibration_gpu_ticks": 5212041.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4533375.0, "compute_span_ns": 4533375.0, "command_buffer_ns": 4533375.03131479, "calibration_cpu_ns": 5244292.0, "calibration_gpu_ticks": 5244292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4491375.0, "compute_span_ns": 4491375.0, "command_buffer_ns": 4491375.002544373, "calibration_cpu_ns": 5122708.0, "calibration_gpu_ticks": 5122708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4486000.0, "compute_span_ns": 4486000.0, "command_buffer_ns": 4485999.990720302, "calibration_cpu_ns": 5164625.0, "calibration_gpu_ticks": 5164625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4444167.0, "compute_span_ns": 4444167.0, "command_buffer_ns": 4444208.345375955, "calibration_cpu_ns": 5154250.0, "calibration_gpu_ticks": 5154250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4480416.0, "compute_span_ns": 4480416.0, "command_buffer_ns": 4480374.976992607, "calibration_cpu_ns": 5176000.0, "calibration_gpu_ticks": 5176000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4478916.0, "compute_span_ns": 4478916.0, "command_buffer_ns": 4478875.023778528, "calibration_cpu_ns": 5248833.0, "calibration_gpu_ticks": 5248833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4507958.0, "compute_span_ns": 4507958.0, "command_buffer_ns": 4507916.630245745, "calibration_cpu_ns": 5338292.0, "calibration_gpu_ticks": 5338292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4530042.0, "compute_span_ns": 4530042.0, "command_buffer_ns": 4530041.71140492, "calibration_cpu_ns": 5340791.0, "calibration_gpu_ticks": 5340791.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 70667.0, "compute_span_ns": 70667.0, "command_buffer_ns": 70666.66148602962, "calibration_cpu_ns": 517666.0, "calibration_gpu_ticks": 517666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70417.0, "compute_span_ns": 70417.0, "command_buffer_ns": 70458.33626762033, "calibration_cpu_ns": 558583.0, "calibration_gpu_ticks": 558583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70250.0, "compute_span_ns": 70250.0, "command_buffer_ns": 70250.01104921103, "calibration_cpu_ns": 551542.0, "calibration_gpu_ticks": 551542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84417.0, "compute_span_ns": 84417.0, "command_buffer_ns": 84458.32645520568, "calibration_cpu_ns": 617458.0, "calibration_gpu_ticks": 617458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71125.0, "compute_span_ns": 71125.0, "command_buffer_ns": 71125.04681572318, "calibration_cpu_ns": 540959.0, "calibration_gpu_ticks": 540959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70250.0, "compute_span_ns": 70250.0, "command_buffer_ns": 70250.01104921103, "calibration_cpu_ns": 537625.0, "calibration_gpu_ticks": 537625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70666.0, "compute_span_ns": 70666.0, "command_buffer_ns": 70624.98480081558, "calibration_cpu_ns": 527959.0, "calibration_gpu_ticks": 527959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 73792.0, "compute_span_ns": 73792.0, "command_buffer_ns": 73791.65617749095, "calibration_cpu_ns": 628125.0, "calibration_gpu_ticks": 628125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72292.0, "compute_span_ns": 72292.0, "command_buffer_ns": 72291.6447557509, "calibration_cpu_ns": 585959.0, "calibration_gpu_ticks": 585959.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 70.205078125, 70.833984375, 70.177734375, 70.09375, 69.440109375, 70.0065, 69.9830625, 70.43684375, 70.78190625 ], "compute_throughput_us_p50": 70.177734375, "compute_throughput_us_p90": 70.792321875, "command_buffer_throughput_us": [ 70.20507837296464, 70.8339848642936, 70.17773441475583, 70.09374985500472, 69.4407553964993, 70.00585901550949, 69.9824222465395, 70.43619734758977, 70.78190174070187 ], "command_buffer_throughput_us_p50": 70.17773441475583, "command_buffer_throughput_us_p90": 70.79231836542021, "compute_latency_us": [ 70.667, 70.417, 70.25, 84.417, 71.125, 70.25, 70.666, 73.792, 72.292 ], "compute_latency_us_p50": 70.667, "compute_latency_us_p90": 75.917, "command_buffer_latency_us": [ 70.66666148602962, 70.45833626762033, 70.25001104921103, 84.45832645520568, 71.12504681572318, 70.25001104921103, 70.62498480081558, 73.79165617749095, 72.2916447557509 ], "command_buffer_latency_us_p50": 70.66666148602962, "command_buffer_latency_us_p90": 75.9249902330339, "counter_control_throughput_ratio": 1.0187861869729673, "counter_control_latency_ratio": 0.9859656145711216 }, "download_ms": 0.847208, "allocation_upload_ms": 15.727125, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.7419583748177121, "device_compute_slowdown": 0.7497890957668865, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "rmsnorm", "m": 1024, "n": 4096, "k": 1 }, "name": "rmsnorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.4065, "warmup_ms": 200.006375, "repetitions": 405, "throughput_us": [ 72.13672839506172, 72.40195308641975, 72.37798518518518, 73.11337283950617, 72.75514320987654, 72.92602716049383, 73.29310864197531, 73.57942469135803, 74.4751012345679 ], "latency_us": [ 356.66700000000003, 316.541, 316.5, 448.458, 316.459, 287.917, 296.58299999999997, 292.208, 297.916 ], "throughput_us_p50": 72.92602716049383, "throughput_us_p90": 73.75856, "latency_us_p50": 316.459, "latency_us_p90": 375.02520000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4532875.027507544, "command_buffers": 1 }, { "command_buffer_ns": 4602874.97844547, "command_buffers": 1 }, { "command_buffer_ns": 4599333.33331719, "command_buffers": 1 }, { "command_buffer_ns": 4448624.968063086, "command_buffers": 1 }, { "command_buffer_ns": 4574291.640892625, "command_buffers": 1 }, { "command_buffer_ns": 4599083.331413567, "command_buffers": 1 }, { "command_buffer_ns": 4686500.004027039, "command_buffers": 1 }, { "command_buffer_ns": 4503458.3308734, "command_buffers": 1 }, { "command_buffer_ns": 4539625.020697713, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 72041.70105978847, "command_buffers": 1 }, { "command_buffer_ns": 86708.34358781576, "command_buffers": 1 }, { "command_buffer_ns": 84750.00504404306, "command_buffers": 1 }, { "command_buffer_ns": 72583.3815522492, "command_buffers": 1 }, { "command_buffer_ns": 70291.62952676415, "command_buffers": 1 }, { "command_buffer_ns": 84208.32455158234, "command_buffers": 1 }, { "command_buffer_ns": 70250.01104921103, "command_buffers": 1 }, { "command_buffer_ns": 71083.31192284822, "command_buffers": 1 }, { "command_buffer_ns": 83666.64405912161, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 70.82617230480537, 71.91992153821047, 71.8645833330811, 69.50976512598572, 71.47330688894726, 71.86067705333699, 73.22656256292248, 70.36653641989687, 70.93164094840176 ], "command_buffer_throughput_us_p50": 71.47330688894726, "command_buffer_throughput_us_p90": 72.18124974315288, "command_buffer_latency_us": [ 72.04170105978847, 86.70834358781576, 84.75000504404306, 72.5833815522492, 70.29162952676415, 84.20832455158234, 70.25001104921103, 71.08331192284822, 83.66664405912161 ], "command_buffer_latency_us_p50": 72.5833815522492, "command_buffer_latency_us_p90": 85.1416727527976 }, "throughput": [ { "compute_ns": 4388458.0, "compute_span_ns": 4388458.0, "command_buffer_ns": 4388458.328321576, "calibration_cpu_ns": 5052667.0, "calibration_gpu_ticks": 5052667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4620625.0, "compute_span_ns": 4620625.0, "command_buffer_ns": 4620624.997187406, "calibration_cpu_ns": 5415375.0, "calibration_gpu_ticks": 5415375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4515125.0, "compute_span_ns": 4515125.0, "command_buffer_ns": 4515125.008765608, "calibration_cpu_ns": 5155333.0, "calibration_gpu_ticks": 5155333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4719417.0, "compute_span_ns": 4719417.0, "command_buffer_ns": 4719458.345789462, "calibration_cpu_ns": 5552791.0, "calibration_gpu_ticks": 5552791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4626708.0, "compute_span_ns": 4626708.0, "command_buffer_ns": 4626666.661351919, "calibration_cpu_ns": 5281958.0, "calibration_gpu_ticks": 5281958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4385208.0, "compute_span_ns": 4385208.0, "command_buffer_ns": 4385166.626889259, "calibration_cpu_ns": 5213833.0, "calibration_gpu_ticks": 5213833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4364250.0, "compute_span_ns": 4364250.0, "command_buffer_ns": 4364249.994978309, "calibration_cpu_ns": 5000292.0, "calibration_gpu_ticks": 5000292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4542958.0, "compute_span_ns": 4542958.0, "command_buffer_ns": 4542916.6639223695, "calibration_cpu_ns": 5400416.0, "calibration_gpu_ticks": 5400416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 4445250.0, "compute_span_ns": 4445250.0, "command_buffer_ns": 4445249.971468002, "calibration_cpu_ns": 5098459.0, "calibration_gpu_ticks": 5098459.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 71458.0, "compute_span_ns": 71458.0, "command_buffer_ns": 71458.3438821137, "calibration_cpu_ns": 535125.0, "calibration_gpu_ticks": 535125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70917.0, "compute_span_ns": 70917.0, "command_buffer_ns": 70916.66338965297, "calibration_cpu_ns": 516667.0, "calibration_gpu_ticks": 516667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84500.0, "compute_span_ns": 84500.0, "command_buffer_ns": 84500.00314041972, "calibration_cpu_ns": 581209.0, "calibration_gpu_ticks": 581209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 71209.0, "compute_span_ns": 71209.0, "command_buffer_ns": 71250.0186637044, "calibration_cpu_ns": 502833.0, "calibration_gpu_ticks": 502833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 82875.0, "compute_span_ns": 82875.0, "command_buffer_ns": 82874.96166303754, "calibration_cpu_ns": 644959.0, "calibration_gpu_ticks": 644959.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84375.0, "compute_span_ns": 84375.0, "command_buffer_ns": 84374.9730847776, "calibration_cpu_ns": 540667.0, "calibration_gpu_ticks": 540667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 72000.0, "compute_span_ns": 72000.0, "command_buffer_ns": 72000.02437457442, "calibration_cpu_ns": 481667.0, "calibration_gpu_ticks": 481667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 83000.0, "compute_span_ns": 83000.0, "command_buffer_ns": 82999.99171867967, "calibration_cpu_ns": 555875.0, "calibration_gpu_ticks": 555875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 84500.0, "compute_span_ns": 84500.0, "command_buffer_ns": 84500.00314041972, "calibration_cpu_ns": 594292.0, "calibration_gpu_ticks": 594292.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 68.56965625, 72.197265625, 70.548828125, 73.740890625, 72.2923125, 68.518875, 68.19140625, 70.98371875, 69.45703125 ], "compute_throughput_us_p50": 70.548828125, "compute_throughput_us_p90": 72.582028125, "command_buffer_throughput_us": [ 68.56966138002463, 72.19726558105322, 70.54882826196263, 73.74153665296035, 72.29166658362374, 68.51822854514467, 68.19140617153607, 70.98307287378702, 69.45703080418753 ], "command_buffer_throughput_us_p50": 70.54882826196263, "command_buffer_throughput_us_p90": 72.58164059749106, "compute_latency_us": [ 71.458, 70.917, 84.5, 71.209, 82.875, 84.375, 72.0, 83.0, 84.5 ], "compute_latency_us_p50": 82.875, "compute_latency_us_p90": 84.5, "command_buffer_latency_us": [ 71.4583438821137, 70.91666338965297, 84.50000314041972, 71.2500186637044, 82.87496166303754, 84.3749730847776, 72.00002437457442, 82.99999171867967, 84.50000314041972 ], "command_buffer_latency_us_p50": 82.87496166303754, "command_buffer_latency_us_p90": 84.50000314041972, "counter_control_throughput_ratio": 0.9816911890378581, "counter_control_latency_ratio": 1.001979003074609 }, "download_ms": 0.663625, "allocation_upload_ms": 1.380083, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 1.8339726848282112e-07, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "rmsnorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-5t6amlpr/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "310a51aaac34ebd04d17becc4645f77793138816428bb43603834b11ab1687fb", "native": { "backend": "metal", "operation": "rmsnorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.64675, "capture_ms": 0.060708, "compile_ms": 51.233667, "allocation_upload_ms": 6.664417, "cold_call_ms": 11.7555, "warmup_ms": 200.317458, "download_ms": 2.959917, "repetitions": 449, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 64.7372873051, 65.3519844098, 65.832311804, 65.6242583519, 65.1752026726, 65.295935412, 65.6803073497, 64.8470668151, 65.5991091314 ], "latency_us": [ 339.75, 274.042, 283.833, 283.042, 270.041, 268.292, 275.458, 326.625, 289.083 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 48, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 32768, "global_write_bytes": 16384, "private_read_bytes": 49152, "private_write_bytes": 16384 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 128, "global_write_bytes": 64, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 1, "reduction_elements": 4096, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 1, "group_barrier_sites_after": 1, "independent_subgroups": false, "normalized_cost": 72, "normalized_kernel_cost": 1152, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 8192, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4159792, "compute_span_ns": 4159792, "calibration_cpu_ns": 4655083, "calibration_gpu_ticks": 4655083, "compute_passes": 1, "command_buffer_ns": 4159875.03948, "command_buffers": 2 }, { "compute_ns": 4222500, "compute_span_ns": 4222500, "calibration_cpu_ns": 4728084, "calibration_gpu_ticks": 4728084, "compute_passes": 1, "command_buffer_ns": 4222625.05163, "command_buffers": 2 }, { "compute_ns": 4283625, "compute_span_ns": 4283625, "calibration_cpu_ns": 4775209, "calibration_gpu_ticks": 4775209, "compute_passes": 1, "command_buffer_ns": 4283750.0223, "command_buffers": 2 }, { "compute_ns": 4312167, "compute_span_ns": 4312167, "calibration_cpu_ns": 4694333, "calibration_gpu_ticks": 4694333, "compute_passes": 1, "command_buffer_ns": 4312250.00648, "command_buffers": 2 }, { "compute_ns": 4107459, "compute_span_ns": 4107459, "calibration_cpu_ns": 4624208, "calibration_gpu_ticks": 4624208, "compute_passes": 1, "command_buffer_ns": 4107625.04907, "command_buffers": 2 }, { "compute_ns": 4303417, "compute_span_ns": 4303417, "calibration_cpu_ns": 4841500, "calibration_gpu_ticks": 4841500, "compute_passes": 1, "command_buffer_ns": 4303458.37958, "command_buffers": 2 }, { "compute_ns": 4236167, "compute_span_ns": 4236167, "calibration_cpu_ns": 4765833, "calibration_gpu_ticks": 4765833, "compute_passes": 1, "command_buffer_ns": 4236333.30502, "command_buffers": 2 }, { "compute_ns": 4239458, "compute_span_ns": 4239458, "calibration_cpu_ns": 4729125, "calibration_gpu_ticks": 4729125, "compute_passes": 1, "command_buffer_ns": 4239541.71129, "command_buffers": 2 }, { "compute_ns": 4197042, "compute_span_ns": 4197042, "calibration_cpu_ns": 4701667, "calibration_gpu_ticks": 4701667, "compute_passes": 1, "command_buffer_ns": 4197208.32724, "command_buffers": 2 } ], "latency": [ { "compute_ns": 65291, "compute_span_ns": 65291, "calibration_cpu_ns": 556292, "calibration_gpu_ticks": 556292, "compute_passes": 1, "command_buffer_ns": 65333.3263472, "command_buffers": 2 }, { "compute_ns": 67459, "compute_span_ns": 67459, "calibration_cpu_ns": 539250, "calibration_gpu_ticks": 539250, "compute_passes": 1, "command_buffer_ns": 67541.6667946, "command_buffers": 2 }, { "compute_ns": 76750, "compute_span_ns": 76750, "calibration_cpu_ns": 840333, "calibration_gpu_ticks": 840333, "compute_passes": 1, "command_buffer_ns": 76791.679021, "command_buffers": 2 }, { "compute_ns": 65042, "compute_span_ns": 65042, "calibration_cpu_ns": 472917, "calibration_gpu_ticks": 472917, "compute_passes": 1, "command_buffer_ns": 65041.705966, "command_buffers": 2 }, { "compute_ns": 76667, "compute_span_ns": 76667, "calibration_cpu_ns": 564167, "calibration_gpu_ticks": 564167, "compute_passes": 1, "command_buffer_ns": 76791.679021, "command_buffers": 2 }, { "compute_ns": 82584, "compute_span_ns": 82584, "calibration_cpu_ns": 567750, "calibration_gpu_ticks": 567750, "compute_passes": 1, "command_buffer_ns": 82624.9597594, "command_buffers": 2 }, { "compute_ns": 65250, "compute_span_ns": 65250, "calibration_cpu_ns": 510167, "calibration_gpu_ticks": 510167, "compute_passes": 1, "command_buffer_ns": 65291.649662, "command_buffers": 2 }, { "compute_ns": 66500, "compute_span_ns": 66500, "calibration_cpu_ns": 505833, "calibration_gpu_ticks": 505833, "compute_passes": 1, "command_buffer_ns": 66499.9824949, "command_buffers": 2 }, { "compute_ns": 73291, "compute_span_ns": 73291, "calibration_cpu_ns": 554667, "calibration_gpu_ticks": 554667, "compute_passes": 1, "command_buffer_ns": 73375.0057407, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4294166.69067, "command_buffers": 2 }, { "command_buffer_ns": 4142374.96443, "command_buffers": 2 }, { "command_buffer_ns": 4181916.65085, "command_buffers": 2 }, { "command_buffer_ns": 4090374.97593, "command_buffers": 2 }, { "command_buffer_ns": 4251333.36103, "command_buffers": 2 }, { "command_buffer_ns": 4205958.27745, "command_buffers": 2 }, { "command_buffer_ns": 4158125.02615, "command_buffers": 2 }, { "command_buffer_ns": 4340791.66735, "command_buffers": 2 }, { "command_buffer_ns": 4092416.66785, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 65333.3263472, "command_buffers": 2 }, { "command_buffer_ns": 65249.9729767, "command_buffers": 2 }, { "command_buffer_ns": 76583.3538026, "command_buffers": 2 }, { "command_buffer_ns": 65125.0011288, "command_buffers": 2 }, { "command_buffer_ns": 65083.3826512, "command_buffers": 2 }, { "command_buffer_ns": 65208.3544992, "command_buffers": 2 }, { "command_buffer_ns": 64749.9691695, "command_buffers": 2 }, { "command_buffer_ns": 77208.3294578, "command_buffers": 2 }, { "command_buffer_ns": 86791.6969582, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 67.09635454171875, 64.72460881921876, 65.34244766953125, 63.912108998906255, 66.42708376609376, 65.71809808515624, 64.97070353359375, 67.82486980234374, 63.94401043515625 ], "command_buffer_throughput_us_p50": 65.34244766953125, "command_buffer_throughput_us_p90": 67.24205759384375, "command_buffer_latency_us": [ 65.3333263472, 65.24997297670001, 76.5833538026, 65.12500112880001, 65.0833826512, 65.2083544992, 64.7499691695, 77.2083294578, 86.7916969582 ], "command_buffer_latency_us_p50": 65.24997297670001, "command_buffer_latency_us_p90": 79.12500295788001 }, "compute_throughput_us": [ 64.99675, 65.9765625, 66.931640625, 67.377609375, 64.179046875, 67.240890625, 66.190109375, 66.24153125, 65.57878125 ], "compute_throughput_us_p50": 66.190109375, "compute_throughput_us_p90": 67.268234375, "command_buffer_throughput_us": [ 64.998047491875, 65.97851643171875, 66.93359409843751, 67.37890635125, 64.18164139171876, 67.24153718093751, 66.1927078909375, 66.24283923890626, 65.581380113125 ], "command_buffer_throughput_us_p50": 66.1927078909375, "command_buffer_throughput_us_p90": 67.269011015, "compute_latency_us": [ 65.291, 67.459, 76.75, 65.042, 76.667, 82.584, 65.25, 66.5, 73.291 ], "compute_latency_us_p50": 67.459, "compute_latency_us_p90": 77.9168, "command_buffer_latency_us": [ 65.3333263472, 67.54166679459999, 76.79167902100001, 65.041705966, 76.79167902100001, 82.6249597594, 65.291649662, 66.49998249490001, 73.3750057407 ], "command_buffer_latency_us_p50": 67.54166679459999, "command_buffer_latency_us_p90": 77.95833516868001, "counter_control_throughput_ratio": 1.019372965482144, "counter_control_latency_ratio": 1.0027202415153686 }, "correctness": { "max_abs_error": 1.5832164268481108e-07, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 65.3519844098, "throughput_us_p90": 65.71070824056, "latency_us_p50": 283.042, "latency_us_p90": 329.25 }, "slowdown": 0.8961407463754324, "device_compute_slowdown": 0.9382169928850255, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-l0wa9snq/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "53243cd32bb4620fa91e934e3d500807c417fa70fc3ac1977dac07575a48be10", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.844459, "capture_ms": 0.072667, "compile_ms": 65.388167, "allocation_upload_ms": 1.298375, "cold_call_ms": 1.973709, "warmup_ms": 200.16875, "download_ms": 0.272125, "repetitions": 4965, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.92341389728, 5.42305317221, 5.37196213494, 5.39360523666, 5.4710306143, 5.45047834844, 5.35303786506, 5.36133756294, 5.40877804632 ], "latency_us": [ 227.541, 230.708, 248, 230.166, 240.125, 224.583, 257.291, 261.917, 238.709 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 338750, "compute_span_ns": 338750, "calibration_cpu_ns": 771083, "calibration_gpu_ticks": 771083, "compute_passes": 1, "command_buffer_ns": 338874.990121, "command_buffers": 2 }, { "compute_ns": 339583, "compute_span_ns": 339583, "calibration_cpu_ns": 792000, "calibration_gpu_ticks": 792000, "compute_passes": 1, "command_buffer_ns": 339583.319146, "command_buffers": 2 }, { "compute_ns": 351292, "compute_span_ns": 351292, "calibration_cpu_ns": 763125, "calibration_gpu_ticks": 763125, "compute_passes": 1, "command_buffer_ns": 351374.910679, "command_buffers": 2 }, { "compute_ns": 362292, "compute_span_ns": 362292, "calibration_cpu_ns": 751292, "calibration_gpu_ticks": 751292, "compute_passes": 1, "command_buffer_ns": 362416.671123, "command_buffers": 2 }, { "compute_ns": 374666, "compute_span_ns": 374666, "calibration_cpu_ns": 781917, "calibration_gpu_ticks": 781917, "compute_passes": 1, "command_buffer_ns": 374708.324671, "command_buffers": 2 }, { "compute_ns": 364541, "compute_span_ns": 364541, "calibration_cpu_ns": 767833, "calibration_gpu_ticks": 767833, "compute_passes": 1, "command_buffer_ns": 364583.276678, "command_buffers": 2 }, { "compute_ns": 355042, "compute_span_ns": 355042, "calibration_cpu_ns": 768834, "calibration_gpu_ticks": 768834, "compute_passes": 1, "command_buffer_ns": 355124.997441, "command_buffers": 2 }, { "compute_ns": 380708, "compute_span_ns": 380708, "calibration_cpu_ns": 811458, "calibration_gpu_ticks": 811458, "compute_passes": 1, "command_buffer_ns": 380708.370358, "command_buffers": 2 }, { "compute_ns": 377333, "compute_span_ns": 377333, "calibration_cpu_ns": 785542, "calibration_gpu_ticks": 785542, "compute_passes": 1, "command_buffer_ns": 377333.315555, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7583, "compute_span_ns": 7583, "calibration_cpu_ns": 469542, "calibration_gpu_ticks": 469542, "compute_passes": 1, "command_buffer_ns": 7625.02895668, "command_buffers": 2 }, { "compute_ns": 7750, "compute_span_ns": 7750, "calibration_cpu_ns": 378583, "calibration_gpu_ticks": 378583, "compute_passes": 1, "command_buffer_ns": 7916.64933786, "command_buffers": 2 }, { "compute_ns": 7584, "compute_span_ns": 7584, "calibration_cpu_ns": 369750, "calibration_gpu_ticks": 369750, "compute_passes": 1, "command_buffer_ns": 7708.38232711, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 369291, "calibration_gpu_ticks": 369291, "compute_passes": 1, "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 375417, "calibration_gpu_ticks": 375417, "compute_passes": 1, "command_buffer_ns": 7708.38232711, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 372500, "calibration_gpu_ticks": 372500, "compute_passes": 1, "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "compute_ns": 7791, "compute_span_ns": 7791, "calibration_cpu_ns": 408542, "calibration_gpu_ticks": 408542, "compute_passes": 1, "command_buffer_ns": 7791.67748988, "command_buffers": 2 }, { "compute_ns": 7708, "compute_span_ns": 7708, "calibration_cpu_ns": 407250, "calibration_gpu_ticks": 407250, "compute_passes": 1, "command_buffer_ns": 7666.64743423, "command_buffers": 2 }, { "compute_ns": 7500, "compute_span_ns": 7500, "calibration_cpu_ns": 370500, "calibration_gpu_ticks": 370500, "compute_passes": 1, "command_buffer_ns": 7583.35227147, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 365791.667718, "command_buffers": 2 }, { "command_buffer_ns": 356958.364137, "command_buffers": 2 }, { "command_buffer_ns": 363541.708793, "command_buffers": 2 }, { "command_buffer_ns": 382874.975912, "command_buffers": 2 }, { "command_buffer_ns": 344208.325259, "command_buffers": 2 }, { "command_buffer_ns": 364999.985322, "command_buffers": 2 }, { "command_buffer_ns": 356416.683644, "command_buffers": 2 }, { "command_buffer_ns": 366291.613318, "command_buffers": 2 }, { "command_buffer_ns": 387500.040233, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7666.58922657, "command_buffers": 2 }, { "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "command_buffer_ns": 7374.9688454, "command_buffers": 2 }, { "command_buffer_ns": 7791.67748988, "command_buffers": 2 }, { "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "command_buffer_ns": 8250.00461191, "command_buffers": 2 }, { "command_buffer_ns": 7583.35227147, "command_buffers": 2 }, { "command_buffer_ns": 7833.35417509, "command_buffers": 2 }, { "command_buffer_ns": 7833.29596743, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 5.71549480809375, 5.577474439640625, 5.680339199890626, 5.9824214986249995, 5.378255082171875, 5.70312477065625, 5.5690106819375, 5.72330645809375, 6.054688128640625 ], "command_buffer_throughput_us_p50": 5.70312477065625, "command_buffer_throughput_us_p90": 5.996874824628125, "command_buffer_latency_us": [ 7.66658922657, 7.45832221583, 7.3749688454, 7.7916774898800005, 7.54167558625, 8.25000461191, 7.58335227147, 7.83335417509, 7.83329596743 ], "command_buffer_latency_us_p50": 7.66658922657, "command_buffer_latency_us_p90": 7.916684262454 }, "compute_throughput_us": [ 5.29296875, 5.305984375, 5.4889375, 5.6608125, 5.85415625, 5.695953125, 5.54753125, 5.9485625, 5.895828125 ], "compute_throughput_us_p50": 5.6608125, "compute_throughput_us_p90": 5.906375000000001, "command_buffer_throughput_us": [ 5.294921720640625, 5.30598936165625, 5.490232979359376, 5.662760486296874, 5.854817572984375, 5.69661369809375, 5.548828085015625, 5.9485682868437495, 5.895833055546874 ], "command_buffer_throughput_us_p50": 5.662760486296874, "command_buffer_throughput_us_p90": 5.906380101806249, "compute_latency_us": [ 7.583, 7.75, 7.584, 7.708, 7.708, 7.708, 7.791, 7.708, 7.5 ], "compute_latency_us_p50": 7.708, "compute_latency_us_p90": 7.7582, "command_buffer_latency_us": [ 7.6250289566800005, 7.91664933786, 7.70838232711, 7.66664743423, 7.70838232711, 7.66664743423, 7.7916774898800005, 7.66664743423, 7.58335227147 ], "command_buffer_latency_us_p50": 7.66664743423, "command_buffer_latency_us_p90": 7.816671859476, "counter_control_throughput_ratio": 0.9737632938776294, "counter_control_latency_ratio": 0.9945790404752657 }, "correctness": { "max_abs_error": 2.1991937471099732e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 5.39360523666, "throughput_us_p90": 5.454588801612, "latency_us_p50": 238.709, "latency_us_p90": 258.2162 }, "torch": { "cold_call_ms": 8.971042, "warmup_ms": 200.333334, "repetitions": 1992, "throughput_us": [ 12.71607279116466, 12.650769578313252, 12.318963353413654, 11.373472891566264, 11.806559738955825, 12.750606425702813, 12.740629016064256, 12.696390060240963, 11.489478915662652 ], "latency_us": [ 339.209, 217.70800000000003, 212.25, 207.334, 198.5, 196.209, 207.083, 291.041, 200.167 ], "throughput_us_p50": 12.650769578313252, "throughput_us_p90": 12.742624497991967, "latency_us_p50": 207.334, "latency_us_p90": 300.6746, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 484333.3153985441, "command_buffers": 1 }, { "command_buffer_ns": 491666.6657663882, "command_buffers": 1 }, { "command_buffer_ns": 483999.96012449265, "command_buffers": 1 }, { "command_buffer_ns": 502458.36609974504, "command_buffers": 1 }, { "command_buffer_ns": 485000.0259466469, "command_buffers": 1 }, { "command_buffer_ns": 483666.66305810213, "command_buffers": 1 }, { "command_buffer_ns": 482250.00500679016, "command_buffers": 1 }, { "command_buffer_ns": 504874.97355788946, "command_buffers": 1 }, { "command_buffer_ns": 483750.0164285302, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10041.636414825916, "command_buffers": 1 }, { "command_buffer_ns": 10333.373211324215, "command_buffers": 1 }, { "command_buffer_ns": 10124.989785254002, "command_buffers": 1 }, { "command_buffer_ns": 10208.343155682087, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 7.567708053102251, 7.682291652599815, 7.562499376945198, 7.850911970308516, 7.578125405416358, 7.557291610282846, 7.535156328231096, 7.888671461842023, 7.558594006695785 ], "command_buffer_throughput_us_p50": 7.567708053102251, "command_buffer_throughput_us_p90": 7.8584638686152175, "command_buffer_latency_us": [ 10.25001984089613, 10.041636414825916, 10.333373211324215, 10.124989785254002, 10.208343155682087, 10.25001984089613, 10.25001984089613, 10.25001984089613, 10.25001984089613 ], "command_buffer_latency_us_p50": 10.25001984089613, "command_buffer_latency_us_p90": 10.266690514981747 }, "throughput": [ { "compute_ns": 543917.0, "compute_span_ns": 543917.0, "command_buffer_ns": 543958.3328552544, "calibration_cpu_ns": 1323083.0, "calibration_gpu_ticks": 1323083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 482167.0, "compute_span_ns": 482167.0, "command_buffer_ns": 482166.6516363621, "calibration_cpu_ns": 1247167.0, "calibration_gpu_ticks": 1247167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483125.0, "compute_span_ns": 483125.0, "command_buffer_ns": 483125.0407733023, "calibration_cpu_ns": 1186791.0, "calibration_gpu_ticks": 1186791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483875.0, "compute_span_ns": 483875.0, "command_buffer_ns": 483874.98827651143, "calibration_cpu_ns": 1179583.0, "calibration_gpu_ticks": 1179583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 529083.0, "compute_span_ns": 529083.0, "command_buffer_ns": 529041.6302159429, "calibration_cpu_ns": 1698667.0, "calibration_gpu_ticks": 1698667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 508291.0, "compute_span_ns": 508291.0, "command_buffer_ns": 508249.9701529741, "calibration_cpu_ns": 1331542.0, "calibration_gpu_ticks": 1331542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 484417.0, "compute_span_ns": 484417.0, "command_buffer_ns": 484416.66876897216, "calibration_cpu_ns": 1203792.0, "calibration_gpu_ticks": 1203792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 483125.0, "compute_span_ns": 483125.0, "command_buffer_ns": 483124.9825656414, "calibration_cpu_ns": 1164375.0, "calibration_gpu_ticks": 1164375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 481667.0, "compute_span_ns": 481667.0, "command_buffer_ns": 481708.32451432943, "calibration_cpu_ns": 1187750.0, "calibration_gpu_ticks": 1187750.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 10166.0, "compute_span_ns": 10166.0, "command_buffer_ns": 10124.989785254002, "calibration_cpu_ns": 404500.0, "calibration_gpu_ticks": 404500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10750.0, "compute_span_ns": 10750.0, "command_buffer_ns": 10750.023648142815, "calibration_cpu_ns": 500000.0, "calibration_gpu_ticks": 500000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10250.0, "compute_span_ns": 10250.0, "command_buffer_ns": 10250.01984089613, "calibration_cpu_ns": 428583.0, "calibration_gpu_ticks": 428583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10333.0, "compute_span_ns": 10333.0, "command_buffer_ns": 10333.373211324215, "calibration_cpu_ns": 463250.0, "calibration_gpu_ticks": 463250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10333.0, "compute_span_ns": 10333.0, "command_buffer_ns": 10291.638318449259, "calibration_cpu_ns": 412667.0, "calibration_gpu_ticks": 412667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10250.0, "compute_span_ns": 10250.0, "command_buffer_ns": 10250.01984089613, "calibration_cpu_ns": 459792.0, "calibration_gpu_ticks": 459792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10292.0, "compute_span_ns": 10292.0, "command_buffer_ns": 10291.696526110172, "calibration_cpu_ns": 424584.0, "calibration_gpu_ticks": 424584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10167.0, "compute_span_ns": 10167.0, "command_buffer_ns": 10208.343155682087, "calibration_cpu_ns": 424125.0, "calibration_gpu_ticks": 424125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10625.0, "compute_span_ns": 10625.0, "command_buffer_ns": 10624.993592500687, "calibration_cpu_ns": 436167.0, "calibration_gpu_ticks": 436167.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 8.498703125, 7.533859375, 7.548828125, 7.560546875, 8.266921875, 7.942046875, 7.569015625, 7.548828125, 7.526046875 ], "compute_throughput_us_p50": 7.560546875, "compute_throughput_us_p90": 8.313278125, "command_buffer_throughput_us": [ 8.49934895086335, 7.533853931818157, 7.548828762082849, 7.560546691820491, 8.266275472124107, 7.941405783640221, 7.56901044951519, 7.548827852588147, 7.526692570536397 ], "command_buffer_throughput_us_p50": 7.560546691820491, "command_buffer_throughput_us_p90": 8.312890167871956, "compute_latency_us": [ 10.166, 10.75, 10.25, 10.333, 10.333, 10.25, 10.292, 10.167, 10.625 ], "compute_latency_us_p50": 10.292, "compute_latency_us_p90": 10.65, "command_buffer_latency_us": [ 10.124989785254002, 10.750023648142815, 10.25001984089613, 10.333373211324215, 10.291638318449259, 10.25001984089613, 10.291696526110172, 10.208343155682087, 10.624993592500687 ], "command_buffer_latency_us_p50": 10.291638318449259, "command_buffer_latency_us_p90": 10.649999603629112, "counter_control_throughput_ratio": 0.9981923152411721, "counter_control_latency_ratio": 1.0040660101990981 }, "download_ms": 0.33425, "allocation_upload_ms": 1.003, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.4263460181826455, "device_compute_slowdown": 0.7487305605786618, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 37, "n": 1537, "k": 1 }, "name": "layernorm_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.270542, "warmup_ms": 200.28225, "repetitions": 1910, "throughput_us": [ 12.940117801047121, 12.821029319371728, 12.809751308900523, 12.79622617801047, 13.167517277486912, 12.552464921465967, 11.999389528795811, 12.721421989528796, 12.680388481675394 ], "latency_us": [ 259.083, 295.625, 224.916, 244.125, 237.875, 240.667, 227.375, 228.834, 223.583 ], "throughput_us_p50": 12.79622617801047, "throughput_us_p90": 12.985597696335079, "latency_us_p50": 237.875, "latency_us_p90": 266.39140000000003, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 560833.3158306777, "command_buffers": 1 }, { "command_buffer_ns": 582999.9572597444, "command_buffers": 1 }, { "command_buffer_ns": 568999.967072159, "command_buffers": 1 }, { "command_buffer_ns": 564999.9948218465, "command_buffers": 1 }, { "command_buffer_ns": 575583.3117291331, "command_buffers": 1 }, { "command_buffer_ns": 560750.0206679106, "command_buffers": 1 }, { "command_buffer_ns": 559041.6840277612, "command_buffers": 1 }, { "command_buffer_ns": 565875.0305883586, "command_buffers": 1 }, { "command_buffer_ns": 564249.9891109765, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 10499.963536858559, "command_buffers": 1 }, { "command_buffer_ns": 10250.01984089613, "command_buffers": 1 }, { "command_buffer_ns": 10708.346962928772, "command_buffers": 1 }, { "command_buffer_ns": 10249.961633235216, "command_buffers": 1 }, { "command_buffer_ns": 10374.991688877344, "command_buffers": 1 }, { "command_buffer_ns": 10416.668374091387, "command_buffers": 1 }, { "command_buffer_ns": 10500.021744519472, "command_buffers": 1 }, { "command_buffer_ns": 10500.021744519472, "command_buffers": 1 }, { "command_buffer_ns": 10166.666470468044, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 8.76302055985434, 9.109374332183506, 8.890624485502485, 8.828124919091351, 8.993489245767705, 8.761719072936103, 8.735026312933769, 8.841797352943104, 8.816406079859007 ], "command_buffer_throughput_us_p50": 8.828124919091351, "command_buffer_throughput_us_p90": 9.016666263050865, "command_buffer_latency_us": [ 10.499963536858559, 10.25001984089613, 10.708346962928772, 10.249961633235216, 10.374991688877344, 10.416668374091387, 10.500021744519472, 10.500021744519472, 10.166666470468044 ], "command_buffer_latency_us_p50": 10.416668374091387, "command_buffer_latency_us_p90": 10.541686788201332 }, "throughput": [ { "compute_ns": 533584.0, "compute_span_ns": 533584.0, "command_buffer_ns": 533625.0178515911, "calibration_cpu_ns": 1302209.0, "calibration_gpu_ticks": 1302209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 566083.0, "compute_span_ns": 566083.0, "command_buffer_ns": 566083.3558067679, "calibration_cpu_ns": 1263750.0, "calibration_gpu_ticks": 1263750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 557166.0, "compute_span_ns": 557166.0, "command_buffer_ns": 557124.9639615417, "calibration_cpu_ns": 1198542.0, "calibration_gpu_ticks": 1198542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 566458.0, "compute_span_ns": 566458.0, "command_buffer_ns": 566458.3295583725, "calibration_cpu_ns": 1173459.0, "calibration_gpu_ticks": 1173459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 543583.0, "compute_span_ns": 543583.0, "command_buffer_ns": 543541.6824184358, "calibration_cpu_ns": 1168500.0, "calibration_gpu_ticks": 1168500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 545500.0, "compute_span_ns": 545500.0, "command_buffer_ns": 545500.0209622085, "calibration_cpu_ns": 1214375.0, "calibration_gpu_ticks": 1214375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 551375.0, "compute_span_ns": 551375.0, "command_buffer_ns": 551375.0365935266, "calibration_cpu_ns": 1191459.0, "calibration_gpu_ticks": 1191459.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 555250.0, "compute_span_ns": 555250.0, "command_buffer_ns": 555249.978788197, "calibration_cpu_ns": 1204000.0, "calibration_gpu_ticks": 1204000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 564500.0, "compute_span_ns": 564500.0, "command_buffer_ns": 564499.9910145998, "calibration_cpu_ns": 1200375.0, "calibration_gpu_ticks": 1200375.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 10250.0, "compute_span_ns": 10250.0, "command_buffer_ns": 10249.961633235216, "calibration_cpu_ns": 377500.0, "calibration_gpu_ticks": 377500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 13875.0, "compute_span_ns": 13875.0, "command_buffer_ns": 13875.01833960414, "calibration_cpu_ns": 410125.0, "calibration_gpu_ticks": 410125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10125.0, "compute_span_ns": 10125.0, "command_buffer_ns": 10125.047992914915, "calibration_cpu_ns": 390542.0, "calibration_gpu_ticks": 390542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10334.0, "compute_span_ns": 10334.0, "command_buffer_ns": 10375.049896538258, "calibration_cpu_ns": 389042.0, "calibration_gpu_ticks": 389042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10416.0, "compute_span_ns": 10416.0, "command_buffer_ns": 10374.991688877344, "calibration_cpu_ns": 414625.0, "calibration_gpu_ticks": 414625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10167.0, "compute_span_ns": 10167.0, "command_buffer_ns": 10208.343155682087, "calibration_cpu_ns": 369541.0, "calibration_gpu_ticks": 369541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10208.0, "compute_span_ns": 10208.0, "command_buffer_ns": 10166.666470468044, "calibration_cpu_ns": 443583.0, "calibration_gpu_ticks": 443583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10541.0, "compute_span_ns": 10541.0, "command_buffer_ns": 10499.963536858559, "calibration_cpu_ns": 406542.0, "calibration_gpu_ticks": 406542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 10375.0, "compute_span_ns": 10375.0, "command_buffer_ns": 10374.991688877344, "calibration_cpu_ns": 462375.0, "calibration_gpu_ticks": 462375.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 8.33725, 8.845046875, 8.70571875, 8.85090625, 8.493484375, 8.5234375, 8.615234375, 8.67578125, 8.8203125 ], "compute_throughput_us_p50": 8.67578125, "compute_throughput_us_p90": 8.84621875, "command_buffer_throughput_us": [ 8.337890903931111, 8.845052434480749, 8.705077561899088, 8.85091139934957, 8.49283878778806, 8.523437827534508, 8.615234946773853, 8.675780918565579, 8.820312359603122 ], "command_buffer_throughput_us_p50": 8.675780918565579, "command_buffer_throughput_us_p90": 8.846224227454513, "compute_latency_us": [ 10.25, 13.875, 10.125, 10.334, 10.416, 10.167, 10.208, 10.541, 10.375 ], "compute_latency_us_p50": 10.334, "compute_latency_us_p90": 11.2078, "command_buffer_latency_us": [ 10.249961633235216, 13.87501833960414, 10.125047992914915, 10.375049896538258, 10.374991688877344, 10.208343155682087, 10.166666470468044, 10.499963536858559, 10.374991688877344 ], "command_buffer_latency_us_p50": 10.374991688877344, "command_buffer_latency_us_p90": 11.174974497407675, "counter_control_throughput_ratio": 0.9791300460495256, "counter_control_latency_ratio": 0.9999944564247265 }, "download_ms": 0.310375, "allocation_upload_ms": 0.895958, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.0790672678193687e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-l8_xg6hw/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "90a0b6dbe0fbfeffc09b3a8db9fc5856f9e2631dc2ce0e6d9ee991918165a6cf", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 16.773667, "capture_ms": 0.076292, "compile_ms": 58.336542, "allocation_upload_ms": 1.496791, "cold_call_ms": 1.928833, "warmup_ms": 200.028, "download_ms": 0.294375, "repetitions": 5847, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.95989396272, 4.96321463999, 4.7014638276, 4.3789122627, 4.53208910552, 4.85045749957, 5.01221429793, 4.77215512228, 4.37186454592 ], "latency_us": [ 286.084, 204.166, 197.791, 216.208, 347.5, 235.667, 228.167, 248.375, 236.834 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 18444, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 283833, "compute_span_ns": 283833, "calibration_cpu_ns": 737583, "calibration_gpu_ticks": 737583, "compute_passes": 1, "command_buffer_ns": 283874.978777, "command_buffers": 2 }, { "compute_ns": 283375, "compute_span_ns": 283375, "calibration_cpu_ns": 724208, "calibration_gpu_ticks": 724208, "compute_passes": 1, "command_buffer_ns": 283416.651655, "command_buffers": 2 }, { "compute_ns": 283208, "compute_span_ns": 283208, "calibration_cpu_ns": 712959, "calibration_gpu_ticks": 712959, "compute_passes": 1, "command_buffer_ns": 283291.679807, "command_buffers": 2 }, { "compute_ns": 283833, "compute_span_ns": 283833, "calibration_cpu_ns": 821750, "calibration_gpu_ticks": 821750, "compute_passes": 1, "command_buffer_ns": 283833.360299, "command_buffers": 2 }, { "compute_ns": 269375, "compute_span_ns": 269375, "calibration_cpu_ns": 713459, "calibration_gpu_ticks": 713459, "compute_passes": 1, "command_buffer_ns": 269374.984782, "command_buffers": 2 }, { "compute_ns": 282792, "compute_span_ns": 282792, "calibration_cpu_ns": 727584, "calibration_gpu_ticks": 727584, "compute_passes": 1, "command_buffer_ns": 282958.382741, "command_buffers": 2 }, { "compute_ns": 267833, "compute_span_ns": 267833, "calibration_cpu_ns": 745375, "calibration_gpu_ticks": 745375, "compute_passes": 1, "command_buffer_ns": 267874.97336, "command_buffers": 2 }, { "compute_ns": 285250, "compute_span_ns": 285250, "calibration_cpu_ns": 707167, "calibration_gpu_ticks": 707167, "compute_passes": 1, "command_buffer_ns": 285291.695036, "command_buffers": 2 }, { "compute_ns": 283667, "compute_span_ns": 283667, "calibration_cpu_ns": 773083, "calibration_gpu_ticks": 773083, "compute_passes": 1, "command_buffer_ns": 283666.711766, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7250, "compute_span_ns": 7250, "calibration_cpu_ns": 419333, "calibration_gpu_ticks": 419333, "compute_passes": 1, "command_buffer_ns": 7333.35036784, "command_buffers": 2 }, { "compute_ns": 7417, "compute_span_ns": 7417, "calibration_cpu_ns": 459041, "calibration_gpu_ticks": 459041, "compute_passes": 1, "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "compute_ns": 7208, "compute_span_ns": 7208, "calibration_cpu_ns": 422250, "calibration_gpu_ticks": 422250, "compute_passes": 1, "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "compute_ns": 7625, "compute_span_ns": 7625, "calibration_cpu_ns": 459708, "calibration_gpu_ticks": 459708, "compute_passes": 1, "command_buffer_ns": 7625.02895668, "command_buffers": 2 }, { "compute_ns": 7417, "compute_span_ns": 7417, "calibration_cpu_ns": 449584, "calibration_gpu_ticks": 449584, "compute_passes": 1, "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "compute_ns": 6959, "compute_span_ns": 6959, "calibration_cpu_ns": 428791, "calibration_gpu_ticks": 428791, "compute_passes": 1, "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "compute_ns": 7125, "compute_span_ns": 7125, "calibration_cpu_ns": 445292, "calibration_gpu_ticks": 445292, "compute_passes": 1, "command_buffer_ns": 7208.37851986, "command_buffers": 2 }, { "compute_ns": 7375, "compute_span_ns": 7375, "calibration_cpu_ns": 410416, "calibration_gpu_ticks": 410416, "compute_passes": 1, "command_buffer_ns": 7375.02705306, "command_buffers": 2 }, { "compute_ns": 7375, "compute_span_ns": 7375, "calibration_cpu_ns": 439459, "calibration_gpu_ticks": 439459, "compute_passes": 1, "command_buffer_ns": 7499.99890104, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 288124.952931, "command_buffers": 2 }, { "command_buffer_ns": 268500.007223, "command_buffers": 2 }, { "command_buffer_ns": 267666.648142, "command_buffers": 2 }, { "command_buffer_ns": 269666.663371, "command_buffers": 2 }, { "command_buffer_ns": 269249.954727, "command_buffers": 2 }, { "command_buffer_ns": 283458.32834, "command_buffers": 2 }, { "command_buffer_ns": 283291.679807, "command_buffers": 2 }, { "command_buffer_ns": 268458.27233, "command_buffers": 2 }, { "command_buffer_ns": 267833.354883, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 7541.67558625, "command_buffers": 2 }, { "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "command_buffer_ns": 7416.58732295, "command_buffers": 2 }, { "command_buffer_ns": 7458.32221583, "command_buffers": 2 }, { "command_buffer_ns": 7541.61737859, "command_buffers": 2 }, { "command_buffer_ns": 7374.9688454, "command_buffers": 2 }, { "command_buffer_ns": 7416.64553061, "command_buffers": 2 }, { "command_buffer_ns": 7833.29596743, "command_buffers": 2 }, { "command_buffer_ns": 7375.02705306, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.501952389546875, 4.195312612859375, 4.182291377218751, 4.2135416151718745, 4.207030542609375, 4.4290363803125, 4.426432496984375, 4.19466050515625, 4.184896170046875 ], "command_buffer_throughput_us_p50": 4.207030542609375, "command_buffer_throughput_us_p90": 4.443619582159375, "command_buffer_latency_us": [ 7.54167558625, 7.45832221583, 7.41658732295, 7.45832221583, 7.54161737859, 7.3749688454, 7.416645530609999, 7.83329596743, 7.37502705306 ], "command_buffer_latency_us_p50": 7.45832221583, "command_buffer_latency_us_p90": 7.599999662486 }, "compute_throughput_us": [ 4.434890625, 4.427734375, 4.425125, 4.434890625, 4.208984375, 4.418625, 4.184890625, 4.45703125, 4.432296875 ], "compute_throughput_us_p50": 4.427734375, "compute_throughput_us_p90": 4.43931875, "command_buffer_throughput_us": [ 4.4355465433906245, 4.428385182109375, 4.426432496984375, 4.434896254671875, 4.20898413721875, 4.421224730328125, 4.18554645875, 4.4576827349374994, 4.432292371343751 ], "command_buffer_throughput_us_p50": 4.428385182109375, "command_buffer_throughput_us_p90": 4.439973781699999, "compute_latency_us": [ 7.25, 7.417, 7.208, 7.625, 7.417, 6.959, 7.125, 7.375, 7.375 ], "compute_latency_us_p50": 7.375, "compute_latency_us_p90": 7.4586, "command_buffer_latency_us": [ 7.3333503678400005, 7.54167558625, 7.208320312200001, 7.6250289566800005, 7.45832221583, 7.0416717790100005, 7.20837851986, 7.37502705306, 7.4999989010400006 ], "command_buffer_latency_us_p50": 7.37502705306, "command_buffer_latency_us_p90": 7.558346260336, "counter_control_throughput_ratio": 1.052534105443022, "counter_control_latency_ratio": 0.9723767992898263 }, "correctness": { "max_abs_error": 2.43199801630567e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 4.77215512228, "throughput_us_p90": 4.973014571578, "latency_us_p50": 235.667, "latency_us_p90": 298.3672 }, "slowdown": 0.37293457116916673, "device_compute_slowdown": 0.510355695632598, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-_yj0cu2t/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "83a70ba0e4d1dab6671da56395e0b13750f8d9c9277e4782c4e371450de131c7", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 18.204833, "capture_ms": 0.078584, "compile_ms": 59.039292, "allocation_upload_ms": 1.977791, "cold_call_ms": 5.1485, "warmup_ms": 200.084, "download_ms": 0.957083, "repetitions": 1305, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 17.1000957854, 17.3565770115, 17.4242015326, 17.3332375479, 16.5785762452, 17.7555877395, 16.8129310345, 17.300702682, 17.3558429119 ], "latency_us": [ 269.292, 332.25, 314, 225.375, 238.208, 290.417, 272.916, 275.209, 264 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1090125, "compute_span_ns": 1090125, "calibration_cpu_ns": 1508542, "calibration_gpu_ticks": 1508542, "compute_passes": 1, "command_buffer_ns": 1090125.00616, "command_buffers": 2 }, { "compute_ns": 1068541, "compute_span_ns": 1068541, "calibration_cpu_ns": 1581375, "calibration_gpu_ticks": 1581375, "compute_passes": 1, "command_buffer_ns": 1068500.04522, "command_buffers": 2 }, { "compute_ns": 1109791, "compute_span_ns": 1109791, "calibration_cpu_ns": 1507250, "calibration_gpu_ticks": 1507250, "compute_passes": 1, "command_buffer_ns": 1109833.36344, "command_buffers": 2 }, { "compute_ns": 1119000, "compute_span_ns": 1119000, "calibration_cpu_ns": 1552750, "calibration_gpu_ticks": 1552750, "compute_passes": 1, "command_buffer_ns": 1119083.31746, "command_buffers": 2 }, { "compute_ns": 1100458, "compute_span_ns": 1100458, "calibration_cpu_ns": 1585792, "calibration_gpu_ticks": 1585792, "compute_passes": 1, "command_buffer_ns": 1100499.93964, "command_buffers": 2 }, { "compute_ns": 1094709, "compute_span_ns": 1094709, "calibration_cpu_ns": 1548667, "calibration_gpu_ticks": 1548667, "compute_passes": 1, "command_buffer_ns": 1094833.36564, "command_buffers": 2 }, { "compute_ns": 1116500, "compute_span_ns": 1116500, "calibration_cpu_ns": 1533458, "calibration_gpu_ticks": 1533458, "compute_passes": 1, "command_buffer_ns": 1116541.67995, "command_buffers": 2 }, { "compute_ns": 1101709, "compute_span_ns": 1101709, "calibration_cpu_ns": 1592334, "calibration_gpu_ticks": 1592334, "compute_passes": 1, "command_buffer_ns": 1101750.00736, "command_buffers": 2 }, { "compute_ns": 1085875, "compute_span_ns": 1085875, "calibration_cpu_ns": 1490708, "calibration_gpu_ticks": 1490708, "compute_passes": 1, "command_buffer_ns": 1085916.65048, "command_buffers": 2 } ], "latency": [ { "compute_ns": 20500, "compute_span_ns": 20500, "calibration_cpu_ns": 482209, "calibration_gpu_ticks": 482209, "compute_passes": 1, "command_buffer_ns": 20499.9814741, "command_buffers": 2 }, { "compute_ns": 23167, "compute_span_ns": 23167, "calibration_cpu_ns": 485709, "calibration_gpu_ticks": 485709, "compute_passes": 1, "command_buffer_ns": 23166.6490436, "command_buffers": 2 }, { "compute_ns": 20458, "compute_span_ns": 20458, "calibration_cpu_ns": 422042, "calibration_gpu_ticks": 422042, "compute_passes": 1, "command_buffer_ns": 20541.5999517, "command_buffers": 2 }, { "compute_ns": 20750, "compute_span_ns": 20750, "calibration_cpu_ns": 462833, "calibration_gpu_ticks": 462833, "compute_passes": 1, "command_buffer_ns": 20791.660063, "command_buffers": 2 }, { "compute_ns": 20667, "compute_span_ns": 20667, "calibration_cpu_ns": 475209, "calibration_gpu_ticks": 475209, "compute_passes": 1, "command_buffer_ns": 20708.3066925, "command_buffers": 2 }, { "compute_ns": 20333, "compute_span_ns": 20333, "calibration_cpu_ns": 433250, "calibration_gpu_ticks": 433250, "compute_passes": 1, "command_buffer_ns": 20375.0096262, "command_buffers": 2 }, { "compute_ns": 20291, "compute_span_ns": 20291, "calibration_cpu_ns": 509458, "calibration_gpu_ticks": 509458, "compute_passes": 1, "command_buffer_ns": 20374.9514185, "command_buffers": 2 }, { "compute_ns": 20375, "compute_span_ns": 20375, "calibration_cpu_ns": 454750, "calibration_gpu_ticks": 454750, "compute_passes": 1, "command_buffer_ns": 20416.6863114, "command_buffers": 2 }, { "compute_ns": 20542, "compute_span_ns": 20542, "calibration_cpu_ns": 446625, "calibration_gpu_ticks": 446625, "compute_passes": 1, "command_buffer_ns": 20666.7464226, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1120125.00176, "command_buffers": 2 }, { "command_buffer_ns": 1119916.67654, "command_buffers": 2 }, { "command_buffer_ns": 1108291.61713, "command_buffers": 2 }, { "command_buffer_ns": 1113250.03672, "command_buffers": 2 }, { "command_buffer_ns": 1118708.34371, "command_buffers": 2 }, { "command_buffer_ns": 1075916.63254, "command_buffers": 2 }, { "command_buffer_ns": 1128583.27338, "command_buffers": 2 }, { "command_buffer_ns": 1096083.31695, "command_buffers": 2 }, { "command_buffer_ns": 1035666.67531, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 20458.3629966, "command_buffers": 2 }, { "command_buffer_ns": 23333.3557844, "command_buffers": 2 }, { "command_buffer_ns": 20749.9833778, "command_buffers": 2 }, { "command_buffer_ns": 20750.0415854, "command_buffers": 2 }, { "command_buffer_ns": 20666.688215, "command_buffers": 2 }, { "command_buffer_ns": 20541.6581593, "command_buffers": 2 }, { "command_buffer_ns": 20458.3047889, "command_buffers": 2 }, { "command_buffer_ns": 20083.3310373, "command_buffers": 2 }, { "command_buffer_ns": 19999.9776669, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 17.501953152499997, 17.498698070937497, 17.317056517656248, 17.39453182375, 17.47981787046875, 16.8111973834375, 17.634113646562497, 17.12630182734375, 16.18229180171875 ], "command_buffer_throughput_us_p50": 17.39453182375, "command_buffer_throughput_us_p90": 17.528385251312496, "command_buffer_latency_us": [ 20.458362996600002, 23.3333557844, 20.7499833778, 20.750041585399998, 20.666688214999997, 20.541658159300002, 20.4583047889, 20.0833310373, 19.999977666899998 ], "command_buffer_latency_us_p50": 20.541658159300002, "command_buffer_latency_us_p90": 21.2667044252 }, "compute_throughput_us": [ 17.033203125, 16.695953125, 17.340484375, 17.484375, 17.19465625, 17.104828125, 17.4453125, 17.214203125, 16.966796875 ], "compute_throughput_us_p50": 17.19465625, "compute_throughput_us_p90": 17.453125, "command_buffer_throughput_us": [ 17.03320322125, 16.6953132065625, 17.34114630375, 17.4856768353125, 17.195311556875, 17.106771338125, 17.445963749218752, 17.214843865, 16.96744766375 ], "command_buffer_throughput_us_p50": 17.195311556875, "command_buffer_throughput_us_p90": 17.4539063664375, "compute_latency_us": [ 20.5, 23.167, 20.458, 20.75, 20.667, 20.333, 20.291, 20.375, 20.542 ], "compute_latency_us_p50": 20.5, "compute_latency_us_p90": 21.2334, "command_buffer_latency_us": [ 20.4999814741, 23.1666490436, 20.541599951699997, 20.791660063, 20.7083066925, 20.3750096262, 20.3749514185, 20.4166863114, 20.666746422600003 ], "command_buffer_latency_us_p50": 20.541599951699997, "command_buffer_latency_us_p90": 21.26665785912, "counter_control_throughput_ratio": 1.0013911016614856, "counter_control_latency_ratio": 1.0020057057441891 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 17.3332375479, "throughput_us_p90": 17.49047877398, "latency_us_p50": 272.916, "latency_us_p90": 317.65 }, "torch": { "cold_call_ms": 7.595208, "warmup_ms": 200.178959, "repetitions": 632, "throughput_us": [ 46.17820411392405, 46.90783227848101, 45.585443037974684, 46.40816139240507, 45.44244462025316, 45.83056487341772, 46.38699841772152, 45.06006012658228, 44.561773734177216 ], "latency_us": [ 418.292, 276.83299999999997, 289.875, 251.292, 265.75, 262.33299999999997, 268.75, 282.5, 301.66700000000003 ], "throughput_us_p50": 45.83056487341772, "throughput_us_p90": 46.50809556962026, "latency_us_p50": 276.83299999999997, "latency_us_p90": 324.992, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2573833.3351910114, "command_buffers": 1 }, { "command_buffer_ns": 2568958.3271741867, "command_buffers": 1 }, { "command_buffer_ns": 2584958.332590759, "command_buffers": 1 }, { "command_buffer_ns": 2578124.9860301614, "command_buffers": 1 }, { "command_buffer_ns": 2580208.2964219153, "command_buffers": 1 }, { "command_buffer_ns": 2599208.3246819675, "command_buffers": 1 }, { "command_buffer_ns": 2581125.0088736415, "command_buffers": 1 }, { "command_buffer_ns": 2581499.982625246, "command_buffers": 1 }, { "command_buffer_ns": 2578750.01989305, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 44541.66628420353, "command_buffers": 1 }, { "command_buffer_ns": 44499.98959898949, "command_buffers": 1 }, { "command_buffer_ns": 44291.66438058019, "command_buffers": 1 }, { "command_buffer_ns": 44375.01775100827, "command_buffers": 1 }, { "command_buffer_ns": 43625.012040138245, "command_buffers": 1 }, { "command_buffer_ns": 43750.04209578037, "command_buffers": 1 }, { "command_buffer_ns": 43875.01394376159, "command_buffers": 1 }, { "command_buffer_ns": 44041.662476956844, "command_buffers": 1 }, { "command_buffer_ns": 43875.01394376159, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 40.216145862359554, 40.13997386209667, 40.389973946730606, 40.28320290672127, 40.31575463159243, 40.61263007315574, 40.33007826365065, 40.33593722851947, 40.29296906082891 ], "command_buffer_throughput_us_p50": 40.31575463159243, "command_buffer_throughput_us_p90": 40.43450517201563, "command_buffer_latency_us": [ 44.54166628420353, 44.49998959898949, 44.29166438058019, 44.37501775100827, 43.625012040138245, 43.75004209578037, 43.87501394376159, 44.041662476956844, 43.87501394376159 ], "command_buffer_latency_us_p50": 44.041662476956844, "command_buffer_latency_us_p90": 44.508324936032295 }, "throughput": [ { "compute_ns": 2638709.0, "compute_span_ns": 2638709.0, "command_buffer_ns": 2638750.011101365, "calibration_cpu_ns": 3609208.0, "calibration_gpu_ticks": 3609208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2594125.0, "compute_span_ns": 2594125.0, "command_buffer_ns": 2594125.0496543944, "calibration_cpu_ns": 3413750.0, "calibration_gpu_ticks": 3413750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2580958.0, "compute_span_ns": 2580958.0, "command_buffer_ns": 2580958.3603404462, "calibration_cpu_ns": 3375500.0, "calibration_gpu_ticks": 3375500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2583041.0, "compute_span_ns": 2583041.0, "command_buffer_ns": 2582999.994046986, "calibration_cpu_ns": 3370083.0, "calibration_gpu_ticks": 3370083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2607834.0, "compute_span_ns": 2607834.0, "command_buffer_ns": 2607875.0379383564, "calibration_cpu_ns": 3399916.0, "calibration_gpu_ticks": 3399916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2567125.0, "compute_span_ns": 2567125.0, "command_buffer_ns": 2567125.018686056, "calibration_cpu_ns": 3410167.0, "calibration_gpu_ticks": 3410167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2563542.0, "compute_span_ns": 2563542.0, "command_buffer_ns": 2563541.696872562, "calibration_cpu_ns": 3350542.0, "calibration_gpu_ticks": 3350542.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2578000.0, "compute_span_ns": 2578000.0, "command_buffer_ns": 2578000.01418218, "calibration_cpu_ns": 3365083.0, "calibration_gpu_ticks": 3365083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2573833.0, "compute_span_ns": 2573833.0, "command_buffer_ns": 2573833.3351910114, "calibration_cpu_ns": 3396583.0, "calibration_gpu_ticks": 3396583.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 44709.0, "compute_span_ns": 44709.0, "command_buffer_ns": 44749.99150261283, "calibration_cpu_ns": 501292.0, "calibration_gpu_ticks": 501292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44333.0, "compute_span_ns": 44333.0, "command_buffer_ns": 44333.34106579423, "calibration_cpu_ns": 489334.0, "calibration_gpu_ticks": 489334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44584.0, "compute_span_ns": 44584.0, "command_buffer_ns": 44625.019654631615, "calibration_cpu_ns": 438708.0, "calibration_gpu_ticks": 438708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44500.0, "compute_span_ns": 44500.0, "command_buffer_ns": 44499.98959898949, "calibration_cpu_ns": 446500.0, "calibration_gpu_ticks": 446500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44042.0, "compute_span_ns": 44042.0, "command_buffer_ns": 44041.662476956844, "calibration_cpu_ns": 462709.0, "calibration_gpu_ticks": 462709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44125.0, "compute_span_ns": 44125.0, "command_buffer_ns": 44124.957639724016, "calibration_cpu_ns": 491834.0, "calibration_gpu_ticks": 491834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44459.0, "compute_span_ns": 44459.0, "command_buffer_ns": 44499.98959898949, "calibration_cpu_ns": 432917.0, "calibration_gpu_ticks": 432917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 43709.0, "compute_span_ns": 43709.0, "command_buffer_ns": 43750.04209578037, "calibration_cpu_ns": 429625.0, "calibration_gpu_ticks": 429625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44250.0, "compute_span_ns": 44250.0, "command_buffer_ns": 44250.04590302706, "calibration_cpu_ns": 469000.0, "calibration_gpu_ticks": 469000.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 41.229828125, 40.533203125, 40.32746875, 40.360015625, 40.74740625, 40.111328125, 40.05534375, 40.28125, 40.216140625 ], "compute_throughput_us_p50": 40.32746875, "compute_throughput_us_p90": 40.843890625, "command_buffer_throughput_us": [ 41.23046892345883, 40.53320390084991, 40.32747438031947, 40.35937490698416, 40.74804746778682, 40.11132841696963, 40.055339013633784, 40.281250221596565, 40.216145862359554 ], "command_buffer_throughput_us_p50": 40.32747438031947, "command_buffer_throughput_us_p90": 40.84453175892122, "compute_latency_us": [ 44.709, 44.333, 44.584, 44.5, 44.042, 44.125, 44.459, 43.709, 44.25 ], "compute_latency_us_p50": 44.333, "compute_latency_us_p90": 44.609, "command_buffer_latency_us": [ 44.74999150261283, 44.33334106579423, 44.625019654631615, 44.49998959898949, 44.041662476956844, 44.124957639724016, 44.49998959898949, 43.75004209578037, 44.25004590302706 ], "command_buffer_latency_us_p50": 44.33334106579423, "command_buffer_latency_us_p90": 44.65001402422786, "counter_control_throughput_ratio": 0.9986442113241827, "counter_control_latency_ratio": 1.0075263659361962 }, "download_ms": 0.364958, "allocation_upload_ms": 1.76825, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.3782025727977332, "device_compute_slowdown": 0.42637578759514877, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 256, "n": 3072, "k": 1 }, "name": "layernorm_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.301667, "warmup_ms": 200.489167, "repetitions": 574, "throughput_us": [ 45.01604181184669, 45.97546515679443, 45.90933449477352, 45.72568118466899, 45.46907665505226, 45.3185243902439, 45.827670731707315, 46.02409930313589, 46.42936933797909 ], "latency_us": [ 398.83299999999997, 314.542, 367.66700000000003, 226.792, 274.667, 276.0, 271.0, 264.25, 251.16699999999997 ], "throughput_us_p50": 45.827670731707315, "throughput_us_p90": 46.105153310104534, "latency_us_p50": 274.667, "latency_us_p90": 373.90020000000004, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 2697416.639421135, "command_buffers": 1 }, { "command_buffer_ns": 2683500.002603978, "command_buffers": 1 }, { "command_buffer_ns": 2635041.659232229, "command_buffers": 1 }, { "command_buffer_ns": 2670874.9937824905, "command_buffers": 1 }, { "command_buffer_ns": 2712375.018745661, "command_buffers": 1 }, { "command_buffer_ns": 2632333.373185247, "command_buffers": 1 }, { "command_buffer_ns": 2688125.0087171793, "command_buffers": 1 }, { "command_buffer_ns": 2642125.0076964498, "command_buffers": 1 }, { "command_buffer_ns": 2682291.6697710752, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 44499.98959898949, "command_buffers": 1 }, { "command_buffer_ns": 49166.67239740491, "command_buffers": 1 }, { "command_buffer_ns": 43958.36731418967, "command_buffers": 1 }, { "command_buffer_ns": 44749.99150261283, "command_buffers": 1 }, { "command_buffer_ns": 44875.02155825496, "command_buffers": 1 }, { "command_buffer_ns": 45416.64384305477, "command_buffers": 1 }, { "command_buffer_ns": 44958.37492868304, "command_buffers": 1 }, { "command_buffer_ns": 44875.02155825496, "command_buffers": 1 }, { "command_buffer_ns": 45124.965254217386, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 42.14713499095524, 41.929687540687155, 41.17252592550358, 41.732421777851414, 42.38085966790095, 41.13020895601949, 42.00195326120593, 41.28320324525703, 41.91080734017305 ], "command_buffer_throughput_us_p50": 41.91080734017305, "command_buffer_throughput_us_p90": 42.19387992634438, "command_buffer_latency_us": [ 44.49998959898949, 49.16667239740491, 43.95836731418967, 44.74999150261283, 44.87502155825496, 45.41664384305477, 44.95837492868304, 44.87502155825496, 45.124965254217386 ], "command_buffer_latency_us_p50": 44.87502155825496, "command_buffer_latency_us_p90": 46.1666495539248 }, "throughput": [ { "compute_ns": 2687375.0, "compute_span_ns": 2687375.0, "command_buffer_ns": 2687375.0030063093, "calibration_cpu_ns": 3393291.0, "calibration_gpu_ticks": 3393291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2689500.0, "compute_span_ns": 2689500.0, "command_buffer_ns": 2689499.990083277, "calibration_cpu_ns": 3411125.0, "calibration_gpu_ticks": 3411125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2624708.0, "compute_span_ns": 2624708.0, "command_buffer_ns": 2624708.3442285657, "calibration_cpu_ns": 3384875.0, "calibration_gpu_ticks": 3384875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2680417.0, "compute_span_ns": 2680417.0, "command_buffer_ns": 2680458.3612829447, "calibration_cpu_ns": 3369500.0, "calibration_gpu_ticks": 3369500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2634583.0, "compute_span_ns": 2634583.0, "command_buffer_ns": 2634541.6554249823, "calibration_cpu_ns": 3302875.0, "calibration_gpu_ticks": 3302875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2650250.0, "compute_span_ns": 2650250.0, "command_buffer_ns": 2650249.982252717, "calibration_cpu_ns": 3312916.0, "calibration_gpu_ticks": 3312916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2662583.0, "compute_span_ns": 2662583.0, "command_buffer_ns": 2662583.312485367, "calibration_cpu_ns": 3411834.0, "calibration_gpu_ticks": 3411834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2650000.0, "compute_span_ns": 2650000.0, "command_buffer_ns": 2649999.9803490937, "calibration_cpu_ns": 3324917.0, "calibration_gpu_ticks": 3324917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 2686708.0, "compute_span_ns": 2686708.0, "command_buffer_ns": 2686708.3506658673, "calibration_cpu_ns": 3353667.0, "calibration_gpu_ticks": 3353667.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 45291.0, "compute_span_ns": 45291.0, "command_buffer_ns": 45249.995309859514, "calibration_cpu_ns": 510667.0, "calibration_gpu_ticks": 510667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45042.0, "compute_span_ns": 45042.0, "command_buffer_ns": 45083.34677666426, "calibration_cpu_ns": 478708.0, "calibration_gpu_ticks": 478708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45333.0, "compute_span_ns": 45333.0, "command_buffer_ns": 45291.67199507356, "calibration_cpu_ns": 529875.0, "calibration_gpu_ticks": 529875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44875.0, "compute_span_ns": 44875.0, "command_buffer_ns": 44875.02155825496, "calibration_cpu_ns": 502708.0, "calibration_gpu_ticks": 502708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44875.0, "compute_span_ns": 44875.0, "command_buffer_ns": 44875.02155825496, "calibration_cpu_ns": 532792.0, "calibration_gpu_ticks": 532792.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45292.0, "compute_span_ns": 45292.0, "command_buffer_ns": 45333.3486802876, "calibration_cpu_ns": 486291.0, "calibration_gpu_ticks": 486291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 45209.0, "compute_span_ns": 45209.0, "command_buffer_ns": 45249.995309859514, "calibration_cpu_ns": 487625.0, "calibration_gpu_ticks": 487625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44708.0, "compute_span_ns": 44708.0, "command_buffer_ns": 44666.638132184744, "calibration_cpu_ns": 511584.0, "calibration_gpu_ticks": 511584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 44500.0, "compute_span_ns": 44500.0, "command_buffer_ns": 44499.98959898949, "calibration_cpu_ns": 453667.0, "calibration_gpu_ticks": 453667.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 41.990234375, 42.0234375, 41.0110625, 41.881515625, 41.165359375, 41.41015625, 41.602859375, 41.40625, 41.9798125 ], "compute_throughput_us_p50": 41.602859375, "compute_throughput_us_p90": 41.996875, "command_buffer_throughput_us": [ 41.99023442197358, 42.02343734505121, 41.01106787857134, 41.88216189504601, 41.16471336601535, 41.4101559726987, 41.60286425758386, 41.40624969295459, 41.97981797915418 ], "command_buffer_throughput_us_p50": 41.60286425758386, "command_buffer_throughput_us_p90": 41.99687500658911, "compute_latency_us": [ 45.291, 45.042, 45.333, 44.875, 44.875, 45.292, 45.209, 44.708, 44.5 ], "compute_latency_us_p50": 45.042, "compute_latency_us_p90": 45.300200000000004, "command_buffer_latency_us": [ 45.249995309859514, 45.08334677666426, 45.29167199507356, 44.87502155825496, 44.87502155825496, 45.3333486802876, 45.249995309859514, 44.666638132184744, 44.49998959898949 ], "command_buffer_latency_us_p50": 45.08334677666426, "command_buffer_latency_us_p90": 45.300007332116365, "counter_control_throughput_ratio": 1.001646607244308, "counter_control_latency_ratio": 1.0 }, "download_ms": 0.366041, "allocation_upload_ms": 0.8755, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 2.982737865586671e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-4hr0_ipv/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "b8d9d984f3aaf8bfdb9269f6e0efc630605362c6ff4e852cc112881dfaf5e97b", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.752416, "capture_ms": 0.068708, "compile_ms": 59.838084, "allocation_upload_ms": 1.891833, "cold_call_ms": 9.35025, "warmup_ms": 200.21425, "download_ms": 0.627792, "repetitions": 1558, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 17.3721116816, 17.4114781772, 16.746817715, 16.3635269576, 16.3456623877, 17.2869062901, 17.5697207959, 17.5682233633, 16.3413831836 ], "latency_us": [ 309.75, 244.125, 249.75, 259.541, 216.125, 218.375, 226.167, 233.5, 232.958 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 36864, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 144, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 1013667, "compute_span_ns": 1013667, "calibration_cpu_ns": 1517291, "calibration_gpu_ticks": 1517291, "compute_passes": 1, "command_buffer_ns": 1013791.59605, "command_buffers": 2 }, { "compute_ns": 1065375, "compute_span_ns": 1065375, "calibration_cpu_ns": 1569958, "calibration_gpu_ticks": 1569958, "compute_passes": 1, "command_buffer_ns": 1065416.66901, "command_buffers": 2 }, { "compute_ns": 1069042, "compute_span_ns": 1069042, "calibration_cpu_ns": 1529542, "calibration_gpu_ticks": 1529542, "compute_passes": 1, "command_buffer_ns": 1069208.31604, "command_buffers": 2 }, { "compute_ns": 1015709, "compute_span_ns": 1015709, "calibration_cpu_ns": 1443292, "calibration_gpu_ticks": 1443292, "compute_passes": 1, "command_buffer_ns": 1015833.28797, "command_buffers": 2 }, { "compute_ns": 1018167, "compute_span_ns": 1018167, "calibration_cpu_ns": 1466250, "calibration_gpu_ticks": 1466250, "compute_passes": 1, "command_buffer_ns": 1018250.01184, "command_buffers": 2 }, { "compute_ns": 1016958, "compute_span_ns": 1016958, "calibration_cpu_ns": 1449666, "calibration_gpu_ticks": 1449666, "compute_passes": 1, "command_buffer_ns": 1017041.67901, "command_buffers": 2 }, { "compute_ns": 1014458, "compute_span_ns": 1014458, "calibration_cpu_ns": 1564625, "calibration_gpu_ticks": 1564625, "compute_passes": 1, "command_buffer_ns": 1014541.65997, "command_buffers": 2 }, { "compute_ns": 1111125, "compute_span_ns": 1111125, "calibration_cpu_ns": 1595708, "calibration_gpu_ticks": 1595708, "compute_passes": 1, "command_buffer_ns": 1111208.34481, "command_buffers": 2 }, { "compute_ns": 1088084, "compute_span_ns": 1088084, "calibration_cpu_ns": 1632000, "calibration_gpu_ticks": 1632000, "compute_passes": 1, "command_buffer_ns": 1088249.96278, "command_buffers": 2 } ], "latency": [ { "compute_ns": 17625, "compute_span_ns": 17625, "calibration_cpu_ns": 390791, "calibration_gpu_ticks": 390791, "compute_passes": 1, "command_buffer_ns": 17624.9886863, "command_buffers": 2 }, { "compute_ns": 20208, "compute_span_ns": 20208, "calibration_cpu_ns": 396875, "calibration_gpu_ticks": 396875, "compute_passes": 1, "command_buffer_ns": 20291.5980481, "command_buffers": 2 }, { "compute_ns": 20708, "compute_span_ns": 20708, "calibration_cpu_ns": 475667, "calibration_gpu_ticks": 475667, "compute_passes": 1, "command_buffer_ns": 20708.3066925, "command_buffers": 2 }, { "compute_ns": 19916, "compute_span_ns": 19916, "calibration_cpu_ns": 447875, "calibration_gpu_ticks": 447875, "compute_passes": 1, "command_buffer_ns": 19999.9776669, "command_buffers": 2 }, { "compute_ns": 19875, "compute_span_ns": 19875, "calibration_cpu_ns": 438584, "calibration_gpu_ticks": 438584, "compute_passes": 1, "command_buffer_ns": 19958.3009817, "command_buffers": 2 }, { "compute_ns": 20083, "compute_span_ns": 20083, "calibration_cpu_ns": 433542, "calibration_gpu_ticks": 433542, "compute_passes": 1, "command_buffer_ns": 20083.3310373, "command_buffers": 2 }, { "compute_ns": 20333, "compute_span_ns": 20333, "calibration_cpu_ns": 438083, "calibration_gpu_ticks": 438083, "compute_passes": 1, "command_buffer_ns": 20333.3329409, "command_buffers": 2 }, { "compute_ns": 20459, "compute_span_ns": 20459, "calibration_cpu_ns": 458167, "calibration_gpu_ticks": 458167, "compute_passes": 1, "command_buffer_ns": 20500.0396818, "command_buffers": 2 }, { "compute_ns": 20166, "compute_span_ns": 20166, "calibration_cpu_ns": 416083, "calibration_gpu_ticks": 416083, "compute_passes": 1, "command_buffer_ns": 20208.361093, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1073916.67552, "command_buffers": 2 }, { "command_buffer_ns": 1017541.68281, "command_buffers": 2 }, { "command_buffer_ns": 1014708.3085, "command_buffers": 2 }, { "command_buffer_ns": 1020875.00272, "command_buffers": 2 }, { "command_buffer_ns": 1071000.00605, "command_buffers": 2 }, { "command_buffer_ns": 1060791.66289, "command_buffers": 2 }, { "command_buffer_ns": 1018374.98369, "command_buffers": 2 }, { "command_buffer_ns": 1027458.34738, "command_buffers": 2 }, { "command_buffer_ns": 1015333.34237, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 17791.6372195, "command_buffers": 2 }, { "command_buffer_ns": 19833.270926, "command_buffers": 2 }, { "command_buffer_ns": 20583.3348446, "command_buffers": 2 }, { "command_buffer_ns": 20208.3028853, "command_buffers": 2 }, { "command_buffer_ns": 20083.3310373, "command_buffers": 2 }, { "command_buffer_ns": 19916.6825041, "command_buffers": 2 }, { "command_buffer_ns": 20208.3028853, "command_buffers": 2 }, { "command_buffer_ns": 20166.6844077, "command_buffers": 2 }, { "command_buffer_ns": 20375.0096262, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 16.779948055000002, 15.89908879390625, 15.8548173203125, 15.9511719175, 16.73437509453125, 16.57486973265625, 15.91210912015625, 16.0540366778125, 15.86458347453125 ], "command_buffer_throughput_us_p50": 15.9511719175, "command_buffer_throughput_us_p90": 16.743489686625, "command_buffer_latency_us": [ 17.7916372195, 19.833270926, 20.583334844600003, 20.2083028853, 20.0833310373, 19.916682504100002, 20.2083028853, 20.1666844077, 20.3750096262 ], "command_buffer_latency_us_p50": 20.1666844077, "command_buffer_latency_us_p90": 20.416674669880003 }, "compute_throughput_us": [ 15.838546875, 16.646484375, 16.70378125, 15.870453125, 15.908859375, 15.88996875, 15.85090625, 17.361328125, 17.0013125 ], "compute_throughput_us_p50": 15.908859375, "compute_throughput_us_p90": 17.073315625, "command_buffer_throughput_us": [ 15.840493688281251, 16.64713545328125, 16.706379938125, 15.872395124531248, 15.910156435000001, 15.891276234531249, 15.852213437031251, 17.36263038765625, 17.0039056684375 ], "command_buffer_throughput_us_p50": 15.910156435000001, "command_buffer_throughput_us_p90": 17.07565061228125, "compute_latency_us": [ 17.625, 20.208, 20.708, 19.916, 19.875, 20.083, 20.333, 20.459, 20.166 ], "compute_latency_us_p50": 20.166, "compute_latency_us_p90": 20.5088, "command_buffer_latency_us": [ 17.6249886863, 20.2915980481, 20.7083066925, 19.999977666899998, 19.9583009817, 20.0833310373, 20.3333329409, 20.5000396818, 20.208361093 ], "command_buffer_latency_us_p50": 20.208361093, "command_buffer_latency_us_p90": 20.54169308394, "counter_control_throughput_ratio": 0.9962358426106361, "counter_control_latency_ratio": 1.0060715063347854 }, "correctness": { "max_abs_error": 2.9463829065434766e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 17.2869062901, "throughput_us_p90": 17.56852284982, "latency_us_p50": 233.5, "latency_us_p90": 269.5828 }, "slowdown": 0.37721546860420097, "device_compute_slowdown": 0.3823982200742662, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-drj5pej8/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "2f415c8199d32d1b999ce6dae6674c9bc8bbdf9b0adda641a6d2aee649468028", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.964166, "capture_ms": 0.075875, "compile_ms": 60.223125, "allocation_upload_ms": 6.116125, "cold_call_ms": 7.143917, "warmup_ms": 200.442334, "download_ms": 3.759459, "repetitions": 353, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 72.7245042493, 72.6961756374, 72.6851983003, 72.7221444759, 72.0030708215, 71.931776204, 71.9809943343, 72.3622521246, 72.085101983 ], "latency_us": [ 371.708, 326.25, 271.541, 273.375, 282.833, 295.25, 379.709, 283.583, 296.625 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4563625, "compute_span_ns": 4563625, "calibration_cpu_ns": 5242000, "calibration_gpu_ticks": 5242000, "compute_passes": 1, "command_buffer_ns": 4563708.38219, "command_buffers": 2 }, { "compute_ns": 4619667, "compute_span_ns": 4619667, "calibration_cpu_ns": 5159833, "calibration_gpu_ticks": 5159833, "compute_passes": 1, "command_buffer_ns": 4619791.63811, "command_buffers": 2 }, { "compute_ns": 4586750, "compute_span_ns": 4586750, "calibration_cpu_ns": 5037750, "calibration_gpu_ticks": 5037750, "compute_passes": 1, "command_buffer_ns": 4586875.03124, "command_buffers": 2 }, { "compute_ns": 4607959, "compute_span_ns": 4607959, "calibration_cpu_ns": 5075083, "calibration_gpu_ticks": 5075083, "compute_passes": 1, "command_buffer_ns": 4608124.96021, "command_buffers": 2 }, { "compute_ns": 4544375, "compute_span_ns": 4544375, "calibration_cpu_ns": 5008167, "calibration_gpu_ticks": 5008167, "compute_passes": 1, "command_buffer_ns": 4544458.35203, "command_buffers": 2 }, { "compute_ns": 4664875, "compute_span_ns": 4664875, "calibration_cpu_ns": 5125584, "calibration_gpu_ticks": 5125584, "compute_passes": 1, "command_buffer_ns": 4664916.66157, "command_buffers": 2 }, { "compute_ns": 4659583, "compute_span_ns": 4659583, "calibration_cpu_ns": 5110709, "calibration_gpu_ticks": 5110709, "compute_passes": 1, "command_buffer_ns": 4659583.32643, "command_buffers": 2 }, { "compute_ns": 4652333, "compute_span_ns": 4652333, "calibration_cpu_ns": 5090708, "calibration_gpu_ticks": 5090708, "compute_passes": 1, "command_buffer_ns": 4652416.6828, "command_buffers": 2 }, { "compute_ns": 4519250, "compute_span_ns": 4519250, "calibration_cpu_ns": 5055541, "calibration_gpu_ticks": 5055541, "compute_passes": 1, "command_buffer_ns": 4519375.04113, "command_buffers": 2 } ], "latency": [ { "compute_ns": 74292, "compute_span_ns": 74292, "calibration_cpu_ns": 564291, "calibration_gpu_ticks": 564291, "compute_passes": 1, "command_buffer_ns": 74416.6900404, "command_buffers": 2 }, { "compute_ns": 73250, "compute_span_ns": 73250, "calibration_cpu_ns": 610458, "calibration_gpu_ticks": 610458, "compute_passes": 1, "command_buffer_ns": 73291.5941626, "command_buffers": 2 }, { "compute_ns": 74250, "compute_span_ns": 74250, "calibration_cpu_ns": 500667, "calibration_gpu_ticks": 500667, "compute_passes": 1, "command_buffer_ns": 74333.33667, "command_buffers": 2 }, { "compute_ns": 73083, "compute_span_ns": 73083, "calibration_cpu_ns": 484125, "calibration_gpu_ticks": 484125, "compute_passes": 1, "command_buffer_ns": 73083.3853595, "command_buffers": 2 }, { "compute_ns": 86167, "compute_span_ns": 86167, "calibration_cpu_ns": 578583, "calibration_gpu_ticks": 578583, "compute_passes": 1, "command_buffer_ns": 86249.9582581, "command_buffers": 2 }, { "compute_ns": 82834, "compute_span_ns": 82834, "calibration_cpu_ns": 531583, "calibration_gpu_ticks": 531583, "compute_passes": 1, "command_buffer_ns": 82916.6965559, "command_buffers": 2 }, { "compute_ns": 72292, "compute_span_ns": 72292, "calibration_cpu_ns": 488583, "calibration_gpu_ticks": 488583, "compute_passes": 1, "command_buffer_ns": 72374.9399185, "command_buffers": 2 }, { "compute_ns": 87458, "compute_span_ns": 87458, "calibration_cpu_ns": 644500, "calibration_gpu_ticks": 644500, "compute_passes": 1, "command_buffer_ns": 87500.0259839, "command_buffers": 2 }, { "compute_ns": 73542, "compute_span_ns": 73542, "calibration_cpu_ns": 527083, "calibration_gpu_ticks": 527083, "compute_passes": 1, "command_buffer_ns": 73583.3309591, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4643041.69873, "command_buffers": 2 }, { "command_buffer_ns": 4609708.38321, "command_buffers": 2 }, { "command_buffer_ns": 4611541.6917, "command_buffers": 2 }, { "command_buffer_ns": 4653291.66036, "command_buffers": 2 }, { "command_buffer_ns": 4552124.99946, "command_buffers": 2 }, { "command_buffer_ns": 4637166.62489, "command_buffers": 2 }, { "command_buffer_ns": 4700625.02427, "command_buffers": 2 }, { "command_buffer_ns": 4592083.33632, "command_buffers": 2 }, { "command_buffer_ns": 4604250.07623, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 86791.6387506, "command_buffers": 2 }, { "command_buffer_ns": 75666.6413508, "command_buffers": 2 }, { "command_buffer_ns": 73708.3610147, "command_buffers": 2 }, { "command_buffer_ns": 73874.9513403, "command_buffers": 2 }, { "command_buffer_ns": 73666.6261218, "command_buffers": 2 }, { "command_buffer_ns": 74291.7181924, "command_buffers": 2 }, { "command_buffer_ns": 76541.6771173, "command_buffers": 2 }, { "command_buffer_ns": 83000.0499263, "command_buffers": 2 }, { "command_buffer_ns": 77541.6847318, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 72.54752654265626, 72.02669348765625, 72.0553389328125, 72.707682193125, 71.12695311656249, 72.45572851390624, 73.44726600421875, 71.75130213, 71.94140744109374 ], "command_buffer_throughput_us_p50": 72.0553389328125, "command_buffer_throughput_us_p90": 72.85559895534375, "command_buffer_latency_us": [ 86.7916387506, 75.66664135079999, 73.7083610147, 73.8749513403, 73.6666261218, 74.2917181924, 76.5416771173, 83.0000499263, 77.54168473179999 ], "command_buffer_latency_us_p50": 75.66664135079999, "command_buffer_latency_us_p90": 83.75836769115999 }, "compute_throughput_us": [ 71.306640625, 72.182296875, 71.66796875, 71.999359375, 71.005859375, 72.888671875, 72.805984375, 72.692703125, 70.61328125 ], "compute_throughput_us_p50": 71.999359375, "compute_throughput_us_p90": 72.82252187499999, "command_buffer_throughput_us": [ 71.30794347171876, 72.18424434546874, 71.66992236312501, 72.00195250328126, 71.00716175046874, 72.88932283703126, 72.80598947546875, 72.69401066875, 70.61523501765625 ], "command_buffer_throughput_us_p50": 72.00195250328126, "command_buffer_throughput_us_p90": 72.82265614778125, "compute_latency_us": [ 74.292, 73.25, 74.25, 73.083, 86.167, 82.834, 72.292, 87.458, 73.542 ], "compute_latency_us_p50": 74.25, "compute_latency_us_p90": 86.4252, "command_buffer_latency_us": [ 74.4166900404, 73.29159416259999, 74.33333667000001, 73.0833853595, 86.2499582581, 82.91669655589999, 72.37493991849999, 87.5000259839, 73.5833309591 ], "command_buffer_latency_us_p50": 74.33333667000001, "command_buffer_latency_us_p90": 86.49997180326, "counter_control_throughput_ratio": 0.9946511032298817, "counter_control_latency_ratio": 0.9892850558079732 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 72.3622521246, "throughput_us_p90": 72.72261643057999, "latency_us_p50": 295.25, "latency_us_p90": 373.3082 }, "torch": { "cold_call_ms": 7.400333, "warmup_ms": 201.620792, "repetitions": 108, "throughput_us": [ 278.22608333333335, 276.49923148148144, 275.2341759259259, 274.9961388888889, 274.72106481481484, 275.4050925925926, 274.6982962962963, 274.4706759259259, 274.8734537037037 ], "latency_us": [ 668.4590000000001, 525.208, 518.9169999999999, 565.584, 542.083, 518.417, 506.791, 522.5840000000001, 579.375 ], "throughput_us_p50": 274.9961388888889, "throughput_us_p90": 276.84460185185185, "latency_us_p50": 525.208, "latency_us_p90": 597.1918000000001, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17067333.33412558, "command_buffers": 1 }, { "command_buffer_ns": 17044333.333615214, "command_buffers": 1 }, { "command_buffer_ns": 17051916.68588668, "command_buffers": 1 }, { "command_buffer_ns": 17215916.654095054, "command_buffers": 1 }, { "command_buffer_ns": 17128749.983385205, "command_buffers": 1 }, { "command_buffer_ns": 17075916.63580388, "command_buffers": 1 }, { "command_buffer_ns": 17095875.05320087, "command_buffers": 1 }, { "command_buffer_ns": 17102750.01823902, "command_buffers": 1 }, { "command_buffer_ns": 17112333.327531815, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 271999.97566640377, "command_buffers": 1 }, { "command_buffer_ns": 284124.9806806445, "command_buffers": 1 }, { "command_buffer_ns": 270916.6146814823, "command_buffers": 1 }, { "command_buffer_ns": 273833.34236219525, "command_buffers": 1 }, { "command_buffer_ns": 279375.0027194619, "command_buffers": 1 }, { "command_buffer_ns": 270874.9962039292, "command_buffers": 1 }, { "command_buffer_ns": 271000.0262595713, "command_buffers": 1 }, { "command_buffer_ns": 270999.9680519104, "command_buffers": 1 }, { "command_buffer_ns": 270499.9642446637, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 266.67708334571216, 266.3177083377377, 266.4361982169794, 268.9986977202352, 267.63671849039383, 266.8111974344356, 267.1230477062636, 267.2304690349847, 267.3802082426846 ], "command_buffer_throughput_us_p50": 267.1230477062636, "command_buffer_throughput_us_p90": 267.9091143363621, "command_buffer_latency_us": [ 271.99997566640377, 284.1249806806445, 270.9166146814823, 273.83334236219525, 279.3750027194619, 270.8749962039292, 271.0000262595713, 270.9999680519104, 270.4999642446637 ], "command_buffer_latency_us_p50": 271.0000262595713, "command_buffer_latency_us_p90": 280.32499831169844 }, "throughput": [ { "compute_ns": 17087083.0, "compute_span_ns": 17087083.0, "command_buffer_ns": 17087041.691411287, "calibration_cpu_ns": 18038292.0, "calibration_gpu_ticks": 18038292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17103458.0, "compute_span_ns": 17103458.0, "command_buffer_ns": 17103458.289057016, "calibration_cpu_ns": 18102834.0, "calibration_gpu_ticks": 18102834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17184917.0, "compute_span_ns": 17184917.0, "command_buffer_ns": 17184916.709084064, "calibration_cpu_ns": 18154334.0, "calibration_gpu_ticks": 18154334.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17086250.0, "compute_span_ns": 17086250.0, "command_buffer_ns": 17086249.95080754, "calibration_cpu_ns": 18082834.0, "calibration_gpu_ticks": 18082834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17162583.0, "compute_span_ns": 17162583.0, "command_buffer_ns": 17162541.626021266, "calibration_cpu_ns": 18025167.0, "calibration_gpu_ticks": 18025167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17110708.0, "compute_span_ns": 17110708.0, "command_buffer_ns": 17110708.286054432, "calibration_cpu_ns": 18027416.0, "calibration_gpu_ticks": 18027416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17147291.0, "compute_span_ns": 17147291.0, "command_buffer_ns": 17147250.00783801, "calibration_cpu_ns": 17915750.0, "calibration_gpu_ticks": 17915750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17096209.0, "compute_span_ns": 17096209.0, "command_buffer_ns": 17096249.968744814, "calibration_cpu_ns": 17865708.0, "calibration_gpu_ticks": 17865708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17181292.0, "compute_span_ns": 17181292.0, "command_buffer_ns": 17181333.32906291, "calibration_cpu_ns": 17921208.0, "calibration_gpu_ticks": 17921208.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 270334.0, "compute_span_ns": 270334.0, "command_buffer_ns": 270374.9923966825, "calibration_cpu_ns": 700875.0, "calibration_gpu_ticks": 700875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271291.0, "compute_span_ns": 271291.0, "command_buffer_ns": 271249.96995553374, "calibration_cpu_ns": 731500.0, "calibration_gpu_ticks": 731500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270750.0, "compute_span_ns": 270750.0, "command_buffer_ns": 270750.024355948, "calibration_cpu_ns": 684000.0, "calibration_gpu_ticks": 684000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 274958.0, "compute_span_ns": 274958.0, "command_buffer_ns": 274958.38003233075, "calibration_cpu_ns": 810375.0, "calibration_gpu_ticks": 810375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270958.0, "compute_span_ns": 270958.0, "command_buffer_ns": 270958.3495743573, "calibration_cpu_ns": 655709.0, "calibration_gpu_ticks": 655709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270958.0, "compute_span_ns": 270958.0, "command_buffer_ns": 270958.3495743573, "calibration_cpu_ns": 714458.0, "calibration_gpu_ticks": 714458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271916.0, "compute_span_ns": 271916.0, "command_buffer_ns": 271875.00381842256, "calibration_cpu_ns": 711084.0, "calibration_gpu_ticks": 711084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 279875.0, "compute_span_ns": 279875.0, "command_buffer_ns": 279875.0065267086, "calibration_cpu_ns": 737084.0, "calibration_gpu_ticks": 737084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270917.0, "compute_span_ns": 270917.0, "command_buffer_ns": 270916.6728891432, "calibration_cpu_ns": 689125.0, "calibration_gpu_ticks": 689125.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 266.985671875, 267.24153125, 268.514328125, 266.97265625, 268.165359375, 267.3548125, 267.926421875, 267.128265625, 268.4576875 ], "compute_throughput_us_p50": 267.3548125, "compute_throughput_us_p90": 268.469015625, "command_buffer_throughput_us": [ 266.98502642830135, 267.2415357665159, 268.5143235794385, 266.97265548136784, 268.1647129065823, 267.3548169696005, 267.9257813724689, 267.1289057616377, 268.45833326660795 ], "command_buffer_throughput_us_p50": 267.3548169696005, "command_buffer_throughput_us_p90": 268.46953132917406, "compute_latency_us": [ 270.334, 271.291, 270.75, 274.958, 270.958, 270.958, 271.916, 279.875, 270.917 ], "compute_latency_us_p50": 270.958, "compute_latency_us_p90": 275.94140000000004, "command_buffer_latency_us": [ 270.3749923966825, 271.24996995553374, 270.75002435594797, 274.95838003233075, 270.95834957435727, 270.95834957435727, 271.87500381842256, 279.8750065267086, 270.91667288914323 ], "command_buffer_latency_us_p50": 270.95834957435727, "command_buffer_latency_us_p90": 275.9417053312063, "counter_control_throughput_ratio": 1.0020374689682898, "counter_control_latency_ratio": 1.000307718953747 }, "download_ms": 0.669834, "allocation_upload_ms": 16.375209, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.2631391568513538, "device_compute_slowdown": 0.26930264954553607, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "layernorm", "m": 768, "n": 6144, "k": 1 }, "name": "layernorm_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.543, "warmup_ms": 201.596708, "repetitions": 107, "throughput_us": [ 275.63006542056075, 275.20054205607477, 275.3247663551402, 275.90147663551403, 275.41160747663554, 275.55918691588784, 276.85670093457946, 274.54594392523364, 275.7527289719626 ], "latency_us": [ 656.584, 567.5409999999999, 508.042, 512.8340000000001, 503.95900000000006, 502.75000000000006, 646.8340000000001, 475.75, 503.87499999999994 ], "throughput_us_p50": 275.55918691588784, "throughput_us_p90": 276.0925214953271, "latency_us_p50": 508.042, "latency_us_p90": 648.784, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 17183166.6957587, "command_buffers": 1 }, { "command_buffer_ns": 17231666.657608002, "command_buffers": 1 }, { "command_buffer_ns": 17204416.6829437, "command_buffers": 1 }, { "command_buffer_ns": 17196125.00164658, "command_buffers": 1 }, { "command_buffer_ns": 17179500.02057478, "command_buffers": 1 }, { "command_buffer_ns": 17198916.64106399, "command_buffers": 1 }, { "command_buffer_ns": 17201250.011567026, "command_buffers": 1 }, { "command_buffer_ns": 17082583.33383128, "command_buffers": 1 }, { "command_buffer_ns": 17138874.97317046, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 281000.0441968441, "command_buffers": 1 }, { "command_buffer_ns": 271124.9981075525, "command_buffers": 1 }, { "command_buffer_ns": 270750.024355948, "command_buffers": 1 }, { "command_buffer_ns": 271333.38153362274, "command_buffers": 1 }, { "command_buffer_ns": 271124.9981075525, "command_buffers": 1 }, { "command_buffer_ns": 279583.3279378712, "command_buffers": 1 }, { "command_buffer_ns": 276750.0118352473, "command_buffers": 1 }, { "command_buffer_ns": 270583.3758227527, "command_buffers": 1 }, { "command_buffer_ns": 270500.0224523246, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 268.4869796212297, 269.24479152512504, 268.81901067099534, 268.6894531507278, 268.4296878214809, 268.7330725166248, 268.7695314307348, 266.91536459111376, 267.7949214557884 ], "command_buffer_throughput_us_p50": 268.6894531507278, "command_buffer_throughput_us_p90": 268.9041668418213, "command_buffer_latency_us": [ 281.0000441968441, 271.1249981075525, 270.75002435594797, 271.33338153362274, 271.1249981075525, 279.5833279378712, 276.7500118352473, 270.5833758227527, 270.50002245232463 ], "command_buffer_latency_us_p50": 271.1249981075525, "command_buffer_latency_us_p90": 279.8666711896658 }, "throughput": [ { "compute_ns": 17157541.0, "compute_span_ns": 17157541.0, "command_buffer_ns": 17157500.027678907, "calibration_cpu_ns": 17984042.0, "calibration_gpu_ticks": 17984042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17217000.0, "compute_span_ns": 17217000.0, "command_buffer_ns": 17217000.015079975, "calibration_cpu_ns": 18052458.0, "calibration_gpu_ticks": 18052458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17159291.0, "compute_span_ns": 17159291.0, "command_buffer_ns": 17159250.04100427, "calibration_cpu_ns": 17860958.0, "calibration_gpu_ticks": 17860958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17218334.0, "compute_span_ns": 17218334.0, "command_buffer_ns": 17218374.996446073, "calibration_cpu_ns": 17946333.0, "calibration_gpu_ticks": 17946333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17222375.0, "compute_span_ns": 17222375.0, "command_buffer_ns": 17222375.026904047, "calibration_cpu_ns": 18024291.0, "calibration_gpu_ticks": 18024291.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17168500.0, "compute_span_ns": 17168500.0, "command_buffer_ns": 17168499.995023012, "calibration_cpu_ns": 17963958.0, "calibration_gpu_ticks": 17963958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17279292.0, "compute_span_ns": 17279292.0, "command_buffer_ns": 17279333.318583667, "calibration_cpu_ns": 18102625.0, "calibration_gpu_ticks": 18102625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17221125.0, "compute_span_ns": 17221125.0, "command_buffer_ns": 17221125.01738593, "calibration_cpu_ns": 18022084.0, "calibration_gpu_ticks": 18022084.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 17111834.0, "compute_span_ns": 17111834.0, "command_buffer_ns": 17111875.000409782, "calibration_cpu_ns": 17955417.0, "calibration_gpu_ticks": 17955417.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 272708.0, "compute_span_ns": 272708.0, "command_buffer_ns": 272666.6862145066, "calibration_cpu_ns": 782916.0, "calibration_gpu_ticks": 782916.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270875.0, "compute_span_ns": 270875.0, "command_buffer_ns": 270874.9962039292, "calibration_cpu_ns": 732917.0, "calibration_gpu_ticks": 732917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271542.0, "compute_span_ns": 271542.0, "command_buffer_ns": 271541.70675203204, "calibration_cpu_ns": 695292.0, "calibration_gpu_ticks": 695292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 280291.0, "compute_span_ns": 280291.0, "command_buffer_ns": 280249.98027831316, "calibration_cpu_ns": 750875.0, "calibration_gpu_ticks": 750875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271209.0, "compute_span_ns": 271209.0, "command_buffer_ns": 271250.02816319466, "calibration_cpu_ns": 694125.0, "calibration_gpu_ticks": 694125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271208.0, "compute_span_ns": 271208.0, "command_buffer_ns": 271208.3514779806, "calibration_cpu_ns": 696208.0, "calibration_gpu_ticks": 696208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 274750.0, "compute_span_ns": 274750.0, "command_buffer_ns": 274749.99660626054, "calibration_cpu_ns": 752833.0, "calibration_gpu_ticks": 752833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 271125.0, "compute_span_ns": 271125.0, "command_buffer_ns": 271124.9981075525, "calibration_cpu_ns": 693500.0, "calibration_gpu_ticks": 693500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 270208.0, "compute_span_ns": 270208.0, "command_buffer_ns": 270208.34386348724, "calibration_cpu_ns": 714958.0, "calibration_gpu_ticks": 714958.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 268.086578125, 269.015625, 268.113921875, 269.03646875, 269.099609375, 268.2578125, 269.9889375, 269.080078125, 267.37240625 ], "compute_throughput_us_p50": 269.015625, "compute_throughput_us_p90": 269.277475, "command_buffer_throughput_us": [ 268.0859379324829, 269.0156252356246, 268.1132818906917, 269.0371093194699, 269.0996097953757, 268.25781242223457, 269.9895831028698, 269.08007839665515, 267.37304688140284 ], "command_buffer_throughput_us_p50": 269.0156252356246, "command_buffer_throughput_us_p90": 269.27760445687454, "compute_latency_us": [ 272.708, 270.875, 271.542, 280.291, 271.209, 271.208, 274.75, 271.125, 270.208 ], "compute_latency_us_p50": 271.209, "compute_latency_us_p90": 275.8582, "command_buffer_latency_us": [ 272.6666862145066, 270.8749962039292, 271.54170675203204, 280.24998027831316, 271.25002816319466, 271.2083514779806, 274.74999660626054, 271.1249981075525, 270.20834386348724 ], "command_buffer_latency_us_p50": 271.25002816319466, "command_buffer_latency_us_p90": 275.84999334067106, "counter_control_throughput_ratio": 0.9991488552547207, "counter_control_latency_ratio": 0.9990779090627262 }, "download_ms": 0.709916, "allocation_upload_ms": 1.249875, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.046016576746524e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-ohnmqt4k/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "4f0c4e70dcb06b507aa82e242c8819acd3628d5f79a065eb6c2f6b73ec6f396c", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.248667, "capture_ms": 0.079625, "compile_ms": 58.5775, "allocation_upload_ms": 6.45075, "cold_call_ms": 4.180167, "warmup_ms": 200.984542, "download_ms": 15.021875, "repetitions": 394, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 74.7832055838, 75.2687182741, 74.8873730964, 76.0171319797, 75.7707284264, 75.6498502538, 75.9854060914, 75.124786802, 75.7137258883 ], "latency_us": [ 349.5, 322.583, 303.542, 311.208, 360.208, 286.292, 300.291, 283.167, 307.083 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 73728, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 288, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4789083, "compute_span_ns": 4789083, "calibration_cpu_ns": 5369209, "calibration_gpu_ticks": 5369209, "compute_passes": 1, "command_buffer_ns": 4789124.94145, "command_buffers": 2 }, { "compute_ns": 4845208, "compute_span_ns": 4845208, "calibration_cpu_ns": 5290167, "calibration_gpu_ticks": 5290167, "compute_passes": 1, "command_buffer_ns": 4845333.34384, "command_buffers": 2 }, { "compute_ns": 4812583, "compute_span_ns": 4812583, "calibration_cpu_ns": 5282583, "calibration_gpu_ticks": 5282583, "compute_passes": 1, "command_buffer_ns": 4812750.03403, "command_buffers": 2 }, { "compute_ns": 4897333, "compute_span_ns": 4897333, "calibration_cpu_ns": 5331416, "calibration_gpu_ticks": 5331416, "compute_passes": 1, "command_buffer_ns": 4897416.6275, "command_buffers": 2 }, { "compute_ns": 4819292, "compute_span_ns": 4819292, "calibration_cpu_ns": 5364459, "calibration_gpu_ticks": 5364459, "compute_passes": 1, "command_buffer_ns": 4819291.70201, "command_buffers": 2 }, { "compute_ns": 4812458, "compute_span_ns": 4812458, "calibration_cpu_ns": 5324416, "calibration_gpu_ticks": 5324416, "compute_passes": 1, "command_buffer_ns": 4812583.3855, "command_buffers": 2 }, { "compute_ns": 4824333, "compute_span_ns": 4824333, "calibration_cpu_ns": 5369375, "calibration_gpu_ticks": 5369375, "compute_passes": 1, "command_buffer_ns": 4824375.03524, "command_buffers": 2 }, { "compute_ns": 4778292, "compute_span_ns": 4778292, "calibration_cpu_ns": 5372542, "calibration_gpu_ticks": 5372542, "compute_passes": 1, "command_buffer_ns": 4778291.62264, "command_buffers": 2 }, { "compute_ns": 5063250, "compute_span_ns": 5063250, "calibration_cpu_ns": 7234541, "calibration_gpu_ticks": 7234541, "compute_passes": 1, "command_buffer_ns": 5063291.58088, "command_buffers": 2 } ], "latency": [ { "compute_ns": 84750, "compute_span_ns": 84750, "calibration_cpu_ns": 704250, "calibration_gpu_ticks": 704250, "compute_passes": 1, "command_buffer_ns": 84791.6817293, "command_buffers": 2 }, { "compute_ns": 91166, "compute_span_ns": 91166, "calibration_cpu_ns": 837916, "calibration_gpu_ticks": 837916, "compute_passes": 1, "command_buffer_ns": 91124.9662749, "command_buffers": 2 }, { "compute_ns": 112792, "compute_span_ns": 112792, "calibration_cpu_ns": 791958, "calibration_gpu_ticks": 791958, "compute_passes": 1, "command_buffer_ns": 112791.662104, "command_buffers": 2 }, { "compute_ns": 101709, "compute_span_ns": 101709, "calibration_cpu_ns": 698959, "calibration_gpu_ticks": 698959, "compute_passes": 1, "command_buffer_ns": 101833.313238, "command_buffers": 2 }, { "compute_ns": 75583, "compute_span_ns": 75583, "calibration_cpu_ns": 609000, "calibration_gpu_ticks": 609000, "compute_passes": 1, "command_buffer_ns": 75624.9646656, "command_buffers": 2 }, { "compute_ns": 91208, "compute_span_ns": 91208, "calibration_cpu_ns": 799542, "calibration_gpu_ticks": 799542, "compute_passes": 1, "command_buffer_ns": 91250.0545382, "command_buffers": 2 }, { "compute_ns": 76292, "compute_span_ns": 76292, "calibration_cpu_ns": 650958, "calibration_gpu_ticks": 650958, "compute_passes": 1, "command_buffer_ns": 76291.6170061, "command_buffers": 2 }, { "compute_ns": 116792, "compute_span_ns": 116792, "calibration_cpu_ns": 693709, "calibration_gpu_ticks": 693709, "compute_passes": 1, "command_buffer_ns": 116833.31104, "command_buffers": 2 }, { "compute_ns": 149916, "compute_span_ns": 149916, "calibration_cpu_ns": 1987208, "calibration_gpu_ticks": 1987208, "compute_passes": 1, "command_buffer_ns": 149875.006173, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4856166.66265, "command_buffers": 2 }, { "command_buffer_ns": 4811458.34783, "command_buffers": 2 }, { "command_buffer_ns": 4842333.32099, "command_buffers": 2 }, { "command_buffer_ns": 4840666.66104, "command_buffers": 2 }, { "command_buffer_ns": 4874000.03476, "command_buffers": 2 }, { "command_buffer_ns": 4823083.34904, "command_buffers": 2 }, { "command_buffer_ns": 4833833.31448, "command_buffers": 2 }, { "command_buffer_ns": 4810958.28582, "command_buffers": 2 }, { "command_buffer_ns": 4772583.25554, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 87750.0278875, "command_buffers": 2 }, { "command_buffer_ns": 84749.9468364, "command_buffers": 2 }, { "command_buffer_ns": 75375.0209697, "command_buffers": 2 }, { "command_buffer_ns": 97666.6342467, "command_buffers": 2 }, { "command_buffer_ns": 76291.6752137, "command_buffers": 2 }, { "command_buffer_ns": 135041.656904, "command_buffers": 2 }, { "command_buffer_ns": 98499.993328, "command_buffers": 2 }, { "command_buffer_ns": 147583.370563, "command_buffers": 2 }, { "command_buffer_ns": 148624.996655, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 75.87760410390625, 75.17903668484375, 75.66145814046875, 75.63541657875, 76.156250543125, 75.36067732875, 75.52864553875, 75.1712232159375, 74.5716133678125 ], "command_buffer_throughput_us_p50": 75.52864553875, "command_buffer_throughput_us_p90": 75.93333339175, "command_buffer_latency_us": [ 87.7500278875, 84.7499468364, 75.37502096969999, 97.66663424670001, 76.2916752137, 135.041656904, 98.499993328, 147.58337056300002, 148.624996655 ], "command_buffer_latency_us_p50": 97.66663424670001, "command_buffer_latency_us_p90": 147.79169578140002 }, "compute_throughput_us": [ 74.829421875, 75.706375, 75.196609375, 76.520828125, 75.3014375, 75.19465625, 75.380203125, 74.6608125, 79.11328125 ], "compute_throughput_us_p50": 75.3014375, "compute_throughput_us_p90": 77.03931874999999, "command_buffer_throughput_us": [ 74.83007721015625, 75.7083334975, 75.19921928171874, 76.52213480468751, 75.30143284390626, 75.1966153984375, 75.380859925625, 74.66080660374999, 79.11393095125 ], "command_buffer_throughput_us_p50": 75.30143284390626, "command_buffer_throughput_us_p90": 77.040494034, "compute_latency_us": [ 84.75, 91.166, 112.792, 101.709, 75.583, 91.208, 76.292, 116.792, 149.916 ], "compute_latency_us_p50": 91.208, "compute_latency_us_p90": 123.41680000000001, "command_buffer_latency_us": [ 84.79168172930001, 91.1249662749, 112.791662104, 101.833313238, 75.6249646656, 91.2500545382, 76.2916170061, 116.83331104, 149.875006173 ], "command_buffer_latency_us_p50": 91.2500545382, "command_buffer_latency_us_p90": 123.4416500666, "counter_control_throughput_ratio": 0.9978229769671947, "counter_control_latency_ratio": 0.9912610314790902 }, "correctness": { "max_abs_error": 2.7199924934606656e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 75.6498502538, "throughput_us_p90": 75.99175126905999, "latency_us_p50": 307.083, "latency_us_p90": 351.6416 }, "slowdown": 0.27453212901550433, "device_compute_slowdown": 0.2799147354359064, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-uz5r1khm/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "6caff4b0a979c3104fea9f23f0d5430392f6f4e7d743dab03e9fdb6de1428bf3", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.736708, "capture_ms": 0.072834, "compile_ms": 59.92075, "allocation_upload_ms": 5.8585, "cold_call_ms": 5.628958, "warmup_ms": 200.684916, "download_ms": 3.683833, "repetitions": 468, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 62.8797200855, 62.0661495726, 62.2834764957, 62.5283119658, 62.3776709402, 62.3291495726, 62.0056965812, 62.4007307692, 62.4147970085 ], "latency_us": [ 388.375, 288.458, 280.333, 354.5, 287.584, 262.167, 272.875, 286.833, 262.166 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 1024, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 128, "normalized_kernel_cost": 2048, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3919042, "compute_span_ns": 3919042, "calibration_cpu_ns": 4376958, "calibration_gpu_ticks": 4376958, "compute_passes": 1, "command_buffer_ns": 3919083.33404, "command_buffers": 2 }, { "compute_ns": 3921750, "compute_span_ns": 3921750, "calibration_cpu_ns": 4424708, "calibration_gpu_ticks": 4424708, "compute_passes": 1, "command_buffer_ns": 3921791.62009, "command_buffers": 2 }, { "compute_ns": 3949542, "compute_span_ns": 3949542, "calibration_cpu_ns": 4417916, "calibration_gpu_ticks": 4417916, "compute_passes": 1, "command_buffer_ns": 3949624.95193, "command_buffers": 2 }, { "compute_ns": 3933167, "compute_span_ns": 3933167, "calibration_cpu_ns": 4397167, "calibration_gpu_ticks": 4397167, "compute_passes": 1, "command_buffer_ns": 3933291.64945, "command_buffers": 2 }, { "compute_ns": 3975917, "compute_span_ns": 3975917, "calibration_cpu_ns": 4428917, "calibration_gpu_ticks": 4428917, "compute_passes": 1, "command_buffer_ns": 3976083.36061, "command_buffers": 2 }, { "compute_ns": 3932125, "compute_span_ns": 3932125, "calibration_cpu_ns": 4389458, "calibration_gpu_ticks": 4389458, "compute_passes": 1, "command_buffer_ns": 3932166.66998, "command_buffers": 2 }, { "compute_ns": 3984583, "compute_span_ns": 3984583, "calibration_cpu_ns": 4469709, "calibration_gpu_ticks": 4469709, "compute_passes": 1, "command_buffer_ns": 3984624.98561, "command_buffers": 2 }, { "compute_ns": 3975625, "compute_span_ns": 3975625, "calibration_cpu_ns": 4424875, "calibration_gpu_ticks": 4424875, "compute_passes": 1, "command_buffer_ns": 3975624.97528, "command_buffers": 2 }, { "compute_ns": 3956375, "compute_span_ns": 3956375, "calibration_cpu_ns": 4594541, "calibration_gpu_ticks": 4594541, "compute_passes": 1, "command_buffer_ns": 3956375.00333, "command_buffers": 2 } ], "latency": [ { "compute_ns": 71167, "compute_span_ns": 71167, "calibration_cpu_ns": 557834, "calibration_gpu_ticks": 557834, "compute_passes": 1, "command_buffer_ns": 71291.6371413, "command_buffers": 2 }, { "compute_ns": 70375, "compute_span_ns": 70375, "calibration_cpu_ns": 500125, "calibration_gpu_ticks": 500125, "compute_passes": 1, "command_buffer_ns": 70416.6595824, "command_buffers": 2 }, { "compute_ns": 63250, "compute_span_ns": 63250, "calibration_cpu_ns": 504000, "calibration_gpu_ticks": 504000, "compute_passes": 1, "command_buffer_ns": 63333.3111182, "command_buffers": 2 }, { "compute_ns": 63084, "compute_span_ns": 63084, "calibration_cpu_ns": 459083, "calibration_gpu_ticks": 459083, "compute_passes": 1, "command_buffer_ns": 63208.3392702, "command_buffers": 2 }, { "compute_ns": 70084, "compute_span_ns": 70084, "calibration_cpu_ns": 504625, "calibration_gpu_ticks": 504625, "compute_passes": 1, "command_buffer_ns": 70124.9809936, "command_buffers": 2 }, { "compute_ns": 63583, "compute_span_ns": 63583, "calibration_cpu_ns": 464000, "calibration_gpu_ticks": 464000, "compute_passes": 1, "command_buffer_ns": 63541.6363366, "command_buffers": 2 }, { "compute_ns": 63500, "compute_span_ns": 63500, "calibration_cpu_ns": 431042, "calibration_gpu_ticks": 431042, "compute_passes": 1, "command_buffer_ns": 63500.017859, "command_buffers": 2 }, { "compute_ns": 63917, "compute_span_ns": 63917, "calibration_cpu_ns": 497375, "calibration_gpu_ticks": 497375, "compute_passes": 1, "command_buffer_ns": 63958.3449811, "command_buffers": 2 }, { "compute_ns": 63625, "compute_span_ns": 63625, "calibration_cpu_ns": 436750, "calibration_gpu_ticks": 436750, "compute_passes": 1, "command_buffer_ns": 63666.7245999, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4000375.04733, "command_buffers": 2 }, { "command_buffer_ns": 3933375.06102, "command_buffers": 2 }, { "command_buffer_ns": 3945374.97777, "command_buffers": 2 }, { "command_buffer_ns": 3943999.99641, "command_buffers": 2 }, { "command_buffer_ns": 4001291.64336, "command_buffers": 2 }, { "command_buffer_ns": 3968958.33546, "command_buffers": 2 }, { "command_buffer_ns": 3996541.72361, "command_buffers": 2 }, { "command_buffer_ns": 3958875.02236, "command_buffers": 2 }, { "command_buffer_ns": 3941624.94922, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 63124.9858998, "command_buffers": 2 }, { "command_buffer_ns": 69833.3024047, "command_buffers": 2 }, { "command_buffer_ns": 63208.3974779, "command_buffers": 2 }, { "command_buffer_ns": 63833.3149254, "command_buffers": 2 }, { "command_buffer_ns": 70666.661486, "command_buffers": 2 }, { "command_buffer_ns": 63583.3130218, "command_buffers": 2 }, { "command_buffer_ns": 63791.6382402, "command_buffers": 2 }, { "command_buffer_ns": 70375.0411049, "command_buffers": 2 }, { "command_buffer_ns": 70999.9585524, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 62.50586011453125, 61.4589853284375, 61.64648402765626, 61.62499994390625, 62.520181927500005, 62.014973991562506, 62.44596443140625, 61.857422224375, 61.5878898315625 ], "command_buffer_throughput_us_p50": 61.857422224375, "command_buffer_throughput_us_p90": 62.508724477125, "command_buffer_latency_us": [ 63.124985899799995, 69.8333024047, 63.2083974779, 63.8333149254, 70.666661486, 63.5833130218, 63.7916382402, 70.37504110489999, 70.9999585524 ], "command_buffer_latency_us_p50": 63.8333149254, "command_buffer_latency_us_p90": 70.73332089928 }, "compute_throughput_us": [ 61.23503125, 61.27734375, 61.71159375, 61.455734375, 62.123703125, 61.439453125, 62.259109375, 62.119140625, 61.818359375 ], "compute_throughput_us_p50": 61.71159375, "compute_throughput_us_p90": 62.150784375, "command_buffer_throughput_us": [ 61.235677094375, 61.27799406390625, 61.71288987390625, 61.45768202265625, 62.12630250953125, 61.4401042184375, 62.25976540015625, 62.11914023875, 61.818359427031254 ], "command_buffer_throughput_us_p50": 61.71288987390625, "command_buffer_throughput_us_p90": 62.15299508765625, "compute_latency_us": [ 71.167, 70.375, 63.25, 63.084, 70.084, 63.583, 63.5, 63.917, 63.625 ], "compute_latency_us_p50": 63.625, "compute_latency_us_p90": 70.5334, "command_buffer_latency_us": [ 71.2916371413, 70.4166595824, 63.3333111182, 63.2083392702, 70.12498099359999, 63.5416363366, 63.500017858999996, 63.958344981100005, 63.6667245999 ], "command_buffer_latency_us_p50": 63.6667245999, "command_buffer_latency_us_p90": 70.59165509418, "counter_control_throughput_ratio": 0.9970550886324591, "counter_control_latency_ratio": 0.9954285484862148 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 62.3776709402, "throughput_us_p90": 62.59859358974, "latency_us_p50": 286.833, "latency_us_p90": 361.275 }, "torch": { "cold_call_ms": 6.67925, "warmup_ms": 201.0485, "repetitions": 143, "throughput_us": [ 207.24358741258743, 206.42045454545453, 205.96008391608393, 205.30274125874124, 205.03117482517484, 205.22814685314685, 205.17307692307693, 205.94872027972028, 207.32837762237762 ], "latency_us": [ 511.667, 604.0, 472.292, 450.625, 450.209, 490.125, 463.583, 484.458, 384.625 ], "throughput_us_p50": 205.94872027972028, "throughput_us_p90": 207.26054545454548, "latency_us_p50": 472.292, "latency_us_p90": 530.1336, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 12797624.978702515, "command_buffers": 1 }, { "command_buffer_ns": 12826124.962884933, "command_buffers": 1 }, { "command_buffer_ns": 12802541.663404554, "command_buffers": 1 }, { "command_buffer_ns": 12772999.99492243, "command_buffers": 1 }, { "command_buffer_ns": 12801999.982912093, "command_buffers": 1 }, { "command_buffer_ns": 12684875.051490963, "command_buffers": 1 }, { "command_buffer_ns": 12709333.328530192, "command_buffers": 1 }, { "command_buffer_ns": 12804500.001948327, "command_buffers": 1 }, { "command_buffer_ns": 12712583.295069635, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 203583.33131298423, "command_buffers": 1 }, { "command_buffer_ns": 201374.9908655882, "command_buffers": 1 }, { "command_buffer_ns": 200291.68808832765, "command_buffers": 1 }, { "command_buffer_ns": 200541.689991951, "command_buffers": 1 }, { "command_buffer_ns": 200874.9870583415, "command_buffers": 1 }, { "command_buffer_ns": 201208.34233239293, "command_buffers": 1 }, { "command_buffer_ns": 201750.02282485366, "command_buffers": 1 }, { "command_buffer_ns": 202875.00228732824, "command_buffers": 1 }, { "command_buffer_ns": 202499.97032806277, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 199.9628902922268, 200.40820254507707, 200.03971349069616, 199.57812492066296, 200.03124973300146, 198.2011726795463, 198.58333325828426, 200.0703125304426, 198.63411398546305 ], "command_buffer_throughput_us_p50": 199.9628902922268, "command_buffer_throughput_us_p90": 200.1378905333695, "command_buffer_latency_us": [ 203.58333131298423, 201.3749908655882, 200.29168808832765, 200.541689991951, 200.8749870583415, 201.20834233239293, 201.75002282485366, 202.87500228732824, 202.49997032806277 ], "command_buffer_latency_us_p50": 201.3749908655882, "command_buffer_latency_us_p90": 203.01666809245944 }, "throughput": [ { "compute_ns": 12797958.0, "compute_span_ns": 12797958.0, "command_buffer_ns": 12797916.657291353, "calibration_cpu_ns": 13762625.0, "calibration_gpu_ticks": 13762625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12824167.0, "compute_span_ns": 12824167.0, "command_buffer_ns": 12824166.62434116, "calibration_cpu_ns": 13717292.0, "calibration_gpu_ticks": 13717292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12800500.0, "compute_span_ns": 12800500.0, "command_buffer_ns": 12800500.029698014, "calibration_cpu_ns": 13891958.0, "calibration_gpu_ticks": 13891958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12805750.0, "compute_span_ns": 12805750.0, "command_buffer_ns": 12805750.011466444, "calibration_cpu_ns": 13851208.0, "calibration_gpu_ticks": 13851208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12784625.0, "compute_span_ns": 12784625.0, "command_buffer_ns": 12784624.996129423, "calibration_cpu_ns": 14925709.0, "calibration_gpu_ticks": 14925709.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12660625.0, "compute_span_ns": 12660625.0, "command_buffer_ns": 12660625.041462481, "calibration_cpu_ns": 14644917.0, "calibration_gpu_ticks": 14644917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12733625.0, "compute_span_ns": 12733625.0, "command_buffer_ns": 12733625.015243888, "calibration_cpu_ns": 15298375.0, "calibration_gpu_ticks": 15298375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12700500.0, "compute_span_ns": 12700500.0, "command_buffer_ns": 12700500.02494827, "calibration_cpu_ns": 19169375.0, "calibration_gpu_ticks": 19169375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12704042.0, "compute_span_ns": 12704042.0, "command_buffer_ns": 12704083.346761763, "calibration_cpu_ns": 15133625.0, "calibration_gpu_ticks": 15133625.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 201167.0, "compute_span_ns": 201167.0, "command_buffer_ns": 201166.6656471789, "calibration_cpu_ns": 1072875.0, "calibration_gpu_ticks": 1072875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200375.0, "compute_span_ns": 200375.0, "command_buffer_ns": 200374.98325109482, "calibration_cpu_ns": 824750.0, "calibration_gpu_ticks": 824750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201500.0, "compute_span_ns": 201500.0, "command_buffer_ns": 201500.02092123032, "calibration_cpu_ns": 751875.0, "calibration_gpu_ticks": 751875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200833.0, "compute_span_ns": 200833.0, "command_buffer_ns": 200833.31037312746, "calibration_cpu_ns": 819167.0, "calibration_gpu_ticks": 819167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202209.0, "compute_span_ns": 202209.0, "command_buffer_ns": 202249.96842443943, "calibration_cpu_ns": 1055167.0, "calibration_gpu_ticks": 1055167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 212958.0, "compute_span_ns": 212958.0, "command_buffer_ns": 212958.37359502912, "calibration_cpu_ns": 1369583.0, "calibration_gpu_ticks": 1369583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200750.0, "compute_span_ns": 200750.0, "command_buffer_ns": 200749.95700269938, "calibration_cpu_ns": 719167.0, "calibration_gpu_ticks": 719167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199792.0, "compute_span_ns": 199792.0, "command_buffer_ns": 199791.68428108096, "calibration_cpu_ns": 753625.0, "calibration_gpu_ticks": 753625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201417.0, "compute_span_ns": 201417.0, "command_buffer_ns": 201458.34423601627, "calibration_cpu_ns": 1010666.0, "calibration_gpu_ticks": 1010666.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 199.96809375, 200.377609375, 200.0078125, 200.08984375, 199.759765625, 197.822265625, 198.962890625, 198.4453125, 198.50065625 ], "compute_throughput_us_p50": 199.759765625, "compute_throughput_us_p90": 200.147396875, "command_buffer_throughput_us": [ 199.9674477701774, 200.37760350533063, 200.00781296403147, 200.08984392916318, 199.75976556452224, 197.82226627285127, 198.96289086318575, 198.4453128898167, 198.50130229315255 ], "command_buffer_throughput_us_p50": 199.75976556452224, "command_buffer_throughput_us_p90": 200.14739584439667, "compute_latency_us": [ 201.167, 200.375, 201.5, 200.833, 202.209, 212.958, 200.75, 199.792, 201.417 ], "compute_latency_us_p50": 201.167, "compute_latency_us_p90": 204.3588, "command_buffer_latency_us": [ 201.1666656471789, 200.37498325109482, 201.50002092123032, 200.83331037312746, 202.24996842443943, 212.95837359502912, 200.74995700269938, 199.79168428108096, 201.45834423601627 ], "command_buffer_latency_us_p50": 201.1666656471789, "command_buffer_latency_us_p90": 204.39164945855737, "counter_control_throughput_ratio": 0.9998405290324205, "counter_control_latency_ratio": 0.9950430448128252 }, "download_ms": 1.170791, "allocation_upload_ms": 14.867834, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "slowdown": 0.30287962389607676, "device_compute_slowdown": 0.3089290456309825, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "layernorm", "m": 1024, "n": 4096, "k": 1 }, "name": "layernorm_1024x4096", "backend": "metal", "block": [ 1, 4096, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.599583, "warmup_ms": 201.600833, "repetitions": 143, "throughput_us": [ 204.95658741258742, 205.32167832167832, 205.4018111888112, 205.88024475524475, 206.30594405594405, 207.50758041958045, 206.7695174825175, 205.93851748251748, 206.26427972027972 ], "latency_us": [ 445.666, 432.66700000000003, 502.87499999999994, 438.167, 410.916, 408.08299999999997, 417.084, 459.041, 408.333 ], "throughput_us_p50": 205.93851748251748, "throughput_us_p90": 206.9171300699301, "latency_us_p50": 432.66700000000003, "latency_us_p90": 467.8078, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 12783875.048626214, "command_buffers": 1 }, { "command_buffer_ns": 12699625.047389418, "command_buffers": 1 }, { "command_buffer_ns": 12789916.654583067, "command_buffers": 1 }, { "command_buffer_ns": 12828958.337195218, "command_buffers": 1 }, { "command_buffer_ns": 12797000.003047287, "command_buffers": 1 }, { "command_buffer_ns": 12821500.014979392, "command_buffers": 1 }, { "command_buffer_ns": 12812625.034712255, "command_buffers": 1 }, { "command_buffer_ns": 12819624.971598387, "command_buffers": 1 }, { "command_buffer_ns": 12876541.644800454, "command_buffers": 1 } ], "latency": [ { "command_buffer_ns": 204749.98746067286, "command_buffers": 1 }, { "command_buffer_ns": 200750.0152103603, "command_buffers": 1 }, { "command_buffer_ns": 202041.64320603013, "command_buffers": 1 }, { "command_buffer_ns": 200958.3404287696, "command_buffers": 1 }, { "command_buffer_ns": 200166.65803268552, "command_buffers": 1 }, { "command_buffer_ns": 199958.33281427622, "command_buffers": 1 }, { "command_buffer_ns": 201083.3122767508, "command_buffers": 1 }, { "command_buffer_ns": 200333.30656588078, "command_buffers": 1 }, { "command_buffer_ns": 201208.34233239293, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 199.7480476347846, 198.43164136545965, 199.84244772786042, 200.45247401867528, 199.95312504761387, 200.335937734053, 200.19726616737898, 200.3066401812248, 201.1959632000071 ], "command_buffer_throughput_us_p50": 200.19726616737898, "command_buffer_throughput_us_p90": 200.60117185494164, "command_buffer_latency_us": [ 204.74998746067286, 200.7500152103603, 202.04164320603013, 200.9583404287696, 200.16665803268552, 199.95833281427622, 201.0833122767508, 200.33330656588078, 201.20834233239293 ], "command_buffer_latency_us_p50": 200.9583404287696, "command_buffer_latency_us_p90": 202.58331205695868 }, "throughput": [ { "compute_ns": 12818792.0, "compute_span_ns": 12818792.0, "command_buffer_ns": 12818791.67072475, "calibration_cpu_ns": 13653875.0, "calibration_gpu_ticks": 13653875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12791000.0, "compute_span_ns": 12791000.0, "command_buffer_ns": 12791000.015567988, "calibration_cpu_ns": 13576209.0, "calibration_gpu_ticks": 13576209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12721791.0, "compute_span_ns": 12721791.0, "command_buffer_ns": 12721750.01213327, "calibration_cpu_ns": 13536541.0, "calibration_gpu_ticks": 13536541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12765500.0, "compute_span_ns": 12765500.0, "command_buffer_ns": 12765499.99602139, "calibration_cpu_ns": 13535000.0, "calibration_gpu_ticks": 13535000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12818666.0, "compute_span_ns": 12818666.0, "command_buffer_ns": 12818624.963983893, "calibration_cpu_ns": 13536208.0, "calibration_gpu_ticks": 13536208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12750667.0, "compute_span_ns": 12750667.0, "command_buffer_ns": 12750666.646752506, "calibration_cpu_ns": 13540083.0, "calibration_gpu_ticks": 13540083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12794125.0, "compute_span_ns": 12794125.0, "command_buffer_ns": 12794125.01025945, "calibration_cpu_ns": 13516958.0, "calibration_gpu_ticks": 13516958.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12833625.0, "compute_span_ns": 12833625.0, "command_buffer_ns": 12833625.019993633, "calibration_cpu_ns": 13561166.0, "calibration_gpu_ticks": 13561166.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 12751916.0, "compute_span_ns": 12751916.0, "command_buffer_ns": 12751875.03779307, "calibration_cpu_ns": 13593000.0, "calibration_gpu_ticks": 13593000.0, "compute_passes": 1, "command_buffers": 1 } ], "latency": [ { "compute_ns": 200958.0, "compute_span_ns": 200958.0, "command_buffer_ns": 200958.3404287696, "calibration_cpu_ns": 634250.0, "calibration_gpu_ticks": 634250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200334.0, "compute_span_ns": 200334.0, "command_buffer_ns": 200374.98325109482, "calibration_cpu_ns": 623209.0, "calibration_gpu_ticks": 623209.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 202083.0, "compute_span_ns": 202083.0, "command_buffer_ns": 202041.70141369104, "calibration_cpu_ns": 634416.0, "calibration_gpu_ticks": 634416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201041.0, "compute_span_ns": 201041.0, "command_buffer_ns": 201000.01711398363, "calibration_cpu_ns": 670458.0, "calibration_gpu_ticks": 670458.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201500.0, "compute_span_ns": 201500.0, "command_buffer_ns": 201499.9627135694, "calibration_cpu_ns": 638208.0, "calibration_gpu_ticks": 638208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 200916.0, "compute_span_ns": 200916.0, "command_buffer_ns": 200874.9870583415, "calibration_cpu_ns": 611917.0, "calibration_gpu_ticks": 611917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201417.0, "compute_span_ns": 201417.0, "command_buffer_ns": 201458.34423601627, "calibration_cpu_ns": 657208.0, "calibration_gpu_ticks": 657208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 201208.0, "compute_span_ns": 201208.0, "command_buffer_ns": 201208.34233239293, "calibration_cpu_ns": 640750.0, "calibration_gpu_ticks": 640750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 199917.0, "compute_span_ns": 199917.0, "command_buffer_ns": 199916.65612906218, "calibration_cpu_ns": 629083.0, "calibration_gpu_ticks": 629083.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 200.293625, 199.859375, 198.777984375, 199.4609375, 200.29165625, 199.229171875, 199.908203125, 200.525390625, 199.2486875 ], "compute_throughput_us_p50": 199.859375, "compute_throughput_us_p90": 200.339978125, "command_buffer_throughput_us": [ 200.2936198550742, 199.85937524324981, 198.77734393958235, 199.46093743783422, 200.29101506224833, 199.2291663555079, 199.9082032853039, 200.52539093740052, 199.24804746551672 ], "command_buffer_throughput_us_p50": 199.85937524324981, "command_buffer_throughput_us_p90": 200.33997407153947, "compute_latency_us": [ 200.958, 200.334, 202.083, 201.041, 201.5, 200.916, 201.417, 201.208, 199.917 ], "compute_latency_us_p50": 201.041, "compute_latency_us_p90": 201.6166, "command_buffer_latency_us": [ 200.9583404287696, 200.37498325109482, 202.04170141369104, 201.00001711398363, 201.4999627135694, 200.8749870583415, 201.45834423601627, 201.20834233239293, 199.91665612906218 ], "command_buffer_latency_us_p50": 201.00001711398363, "command_buffer_latency_us_p90": 201.60831045359373, "counter_control_throughput_ratio": 0.99855610974311, "counter_control_latency_ratio": 1.0002073896765127 }, "download_ms": 0.591417, "allocation_upload_ms": 1.235458, "device": "mps:0", "output_policy": "framework_return_value", "correctness": { "max_abs_error": 3.1634934849478213e-07, "atol": 1e-05, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "layernorm", "1024", "4096", "1", "1", "4096", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-qzwaf1gn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7d02ca31ce0660bb7b8cf2d4b7c68e2de8afe58cfb625e16c6aa93173ce8f234", "native": { "backend": "metal", "operation": "layernorm", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.390375, "capture_ms": 0.0685, "compile_ms": 58.161375, "allocation_upload_ms": 5.536084, "cold_call_ms": 7.166917, "warmup_ms": 200.777917, "download_ms": 3.901667, "repetitions": 323, "output_elements": 4194304, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 76.0839783282, 76.6306749226, 76.2769628483, 76.1715665635, 76.1585417957, 76.3707430341, 76.7202012384, 76.7969566563, 76.5735294118 ], "latency_us": [ 399.541, 316.916, 311.458, 336.042, 330.792, 318.625, 319.958, 318.125, 353.25 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 1024, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 512, "reduction_scalar_rounds": 80, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 49152, "global_write_bytes": 16384, "private_read_bytes": 98304, "private_write_bytes": 32768 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 192, "global_write_bytes": 64, "private_read_bytes": 384, "private_write_bytes": 128 }, "striped_storage_scalars_per_worker": 32, "reduction_operations": 2, "reduction_elements": 8192, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 120, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 12288, "fragment_scalars_per_lane": 0, "concurrent_waves": 16, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4914750, "compute_span_ns": 4914750, "calibration_cpu_ns": 5385750, "calibration_gpu_ticks": 5385750, "compute_passes": 1, "command_buffer_ns": 4914749.99581, "command_buffers": 2 }, { "compute_ns": 4861083, "compute_span_ns": 4861083, "calibration_cpu_ns": 5380333, "calibration_gpu_ticks": 5380333, "compute_passes": 1, "command_buffer_ns": 4861083.34735, "command_buffers": 2 }, { "compute_ns": 4883417, "compute_span_ns": 4883417, "calibration_cpu_ns": 5386500, "calibration_gpu_ticks": 5386500, "compute_passes": 1, "command_buffer_ns": 4883500.04889, "command_buffers": 2 }, { "compute_ns": 4921583, "compute_span_ns": 4921583, "calibration_cpu_ns": 5384834, "calibration_gpu_ticks": 5384834, "compute_passes": 1, "command_buffer_ns": 4921624.96084, "command_buffers": 2 }, { "compute_ns": 4886083, "compute_span_ns": 4886083, "calibration_cpu_ns": 5502167, "calibration_gpu_ticks": 5502167, "compute_passes": 1, "command_buffer_ns": 4886208.33494, "command_buffers": 2 }, { "compute_ns": 4913666, "compute_span_ns": 4913666, "calibration_cpu_ns": 5430125, "calibration_gpu_ticks": 5430125, "compute_passes": 1, "command_buffer_ns": 4913749.92998, "command_buffers": 2 }, { "compute_ns": 4847459, "compute_span_ns": 4847459, "calibration_cpu_ns": 5387333, "calibration_gpu_ticks": 5387333, "compute_passes": 1, "command_buffer_ns": 4847583.36097, "command_buffers": 2 }, { "compute_ns": 4879333, "compute_span_ns": 4879333, "calibration_cpu_ns": 5348375, "calibration_gpu_ticks": 5348375, "compute_passes": 1, "command_buffer_ns": 4879333.3699, "command_buffers": 2 }, { "compute_ns": 4886625, "compute_span_ns": 4886625, "calibration_cpu_ns": 5506500, "calibration_gpu_ticks": 5506500, "compute_passes": 1, "command_buffer_ns": 4886666.66206, "command_buffers": 2 } ], "latency": [ { "compute_ns": 77209, "compute_span_ns": 77209, "calibration_cpu_ns": 519000, "calibration_gpu_ticks": 519000, "compute_passes": 1, "command_buffer_ns": 77291.6828282, "command_buffers": 2 }, { "compute_ns": 76667, "compute_span_ns": 76667, "calibration_cpu_ns": 504292, "calibration_gpu_ticks": 504292, "compute_passes": 1, "command_buffer_ns": 76666.707173, "command_buffers": 2 }, { "compute_ns": 77250, "compute_span_ns": 77250, "calibration_cpu_ns": 534667, "calibration_gpu_ticks": 534667, "compute_passes": 1, "command_buffer_ns": 77333.3595134, "command_buffers": 2 }, { "compute_ns": 77208, "compute_span_ns": 77208, "calibration_cpu_ns": 502792, "calibration_gpu_ticks": 502792, "compute_passes": 1, "command_buffer_ns": 77291.6828282, "command_buffers": 2 }, { "compute_ns": 85375, "compute_span_ns": 85375, "calibration_cpu_ns": 644375, "calibration_gpu_ticks": 644375, "compute_passes": 1, "command_buffer_ns": 85458.3340697, "command_buffers": 2 }, { "compute_ns": 85750, "compute_span_ns": 85750, "calibration_cpu_ns": 571375, "calibration_gpu_ticks": 571375, "compute_passes": 1, "command_buffer_ns": 85791.6311361, "command_buffers": 2 }, { "compute_ns": 77625, "compute_span_ns": 77625, "calibration_cpu_ns": 526250, "calibration_gpu_ticks": 526250, "compute_passes": 1, "command_buffer_ns": 77625.0381023, "command_buffers": 2 }, { "compute_ns": 76916, "compute_span_ns": 76916, "calibration_cpu_ns": 526250, "calibration_gpu_ticks": 526250, "compute_passes": 1, "command_buffer_ns": 76999.9460317, "command_buffers": 2 }, { "compute_ns": 75750, "compute_span_ns": 75750, "calibration_cpu_ns": 504792, "calibration_gpu_ticks": 504792, "compute_passes": 1, "command_buffer_ns": 75750.0529289, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4891625.02345, "command_buffers": 2 }, { "command_buffer_ns": 4881541.65214, "command_buffers": 2 }, { "command_buffer_ns": 4938333.35349, "command_buffers": 2 }, { "command_buffer_ns": 4914874.96765, "command_buffers": 2 }, { "command_buffer_ns": 4872041.63801, "command_buffers": 2 }, { "command_buffer_ns": 4898875.02044, "command_buffers": 2 }, { "command_buffer_ns": 4869500.0005, "command_buffers": 2 }, { "command_buffer_ns": 4788874.99776, "command_buffers": 2 }, { "command_buffer_ns": 4848416.66184, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 77624.9798946, "command_buffers": 2 }, { "command_buffer_ns": 76583.2955949, "command_buffers": 2 }, { "command_buffer_ns": 76249.9985285, "command_buffers": 2 }, { "command_buffer_ns": 78375.0438131, "command_buffers": 2 }, { "command_buffer_ns": 94208.3424889, "command_buffers": 2 }, { "command_buffer_ns": 83583.2906887, "command_buffers": 2 }, { "command_buffer_ns": 77291.6828282, "command_buffers": 2 }, { "command_buffer_ns": 79708.348494, "command_buffers": 2 }, { "command_buffer_ns": 75666.6413508, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 76.43164099140625, 76.2740883146875, 77.16145864828124, 76.79492136953125, 76.12565059390624, 76.544922194375, 76.0859375078125, 74.82617184, 75.75651034125 ], "command_buffer_throughput_us_p50": 76.2740883146875, "command_buffer_throughput_us_p90": 76.86822882528125, "command_buffer_latency_us": [ 77.62497989459999, 76.58329559490001, 76.2499985285, 78.37504381309999, 94.2083424889, 83.5832906887, 77.29168282820001, 79.708348494, 75.66664135079999 ], "command_buffer_latency_us_p50": 77.62497989459999, "command_buffer_latency_us_p90": 85.70830104874001 }, "compute_throughput_us": [ 76.79296875, 75.954421875, 76.303390625, 76.899734375, 76.345046875, 76.77603125, 75.741546875, 76.239578125, 76.353515625 ], "compute_throughput_us_p50": 76.345046875, "compute_throughput_us_p90": 76.814321875, "command_buffer_throughput_us": [ 76.79296868453126, 75.95442730234376, 76.30468826390626, 76.90039001312499, 76.3470052334375, 76.77734265593749, 75.74349001515624, 76.2395839046875, 76.3541665946875 ], "command_buffer_throughput_us_p50": 76.3470052334375, "command_buffer_throughput_us_p90": 76.81445295025, "compute_latency_us": [ 77.209, 76.667, 77.25, 77.208, 85.375, 85.75, 77.625, 76.916, 75.75 ], "compute_latency_us_p50": 77.209, "compute_latency_us_p90": 85.45, "command_buffer_latency_us": [ 77.29168282820001, 76.666707173, 77.33335951340001, 77.29168282820001, 85.4583340697, 85.7916311361, 77.6250381023, 76.9999460317, 75.75005292889999 ], "command_buffer_latency_us_p50": 77.29168282820001, "command_buffer_latency_us_p90": 85.52499348298, "counter_control_throughput_ratio": 1.0029077536652145, "counter_control_latency_ratio": 1.0010891615129913 }, "correctness": { "max_abs_error": 2.827305296548843e-07, "atol": 1e-05, "rtol": 2e-05 }, "throughput_us_p50": 76.3707430341, "throughput_us_p90": 76.73555232198, "latency_us_p50": 319.958, "latency_us_p90": 362.5082 }, "slowdown": 0.37084244350056206, "device_compute_slowdown": 0.3819938237823469, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-0pg3lc43/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "d45e9e72129e03853b1c708b1bdcbc59be9547b0d0e02ca6bdc2eb0b41c08926", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.135458, "capture_ms": 0.07175, "compile_ms": 57.137, "allocation_upload_ms": 1.172167, "cold_call_ms": 1.9875, "warmup_ms": 200.03175, "download_ms": 0.288916, "repetitions": 4859, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.75689442272, 4.60747252521, 4.85538876312, 4.92922062153, 4.91179604857, 4.97688145709, 4.66138444124, 4.76925108047, 4.99569520477 ], "latency_us": [ 221.125, 184.792, 197.167, 201.417, 312.208, 198.667, 224.917, 222.792, 207.166 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 19, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 80, "normalized_kernel_cost": 80, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 313541, "compute_span_ns": 313541, "calibration_cpu_ns": 757125, "calibration_gpu_ticks": 757125, "compute_passes": 1, "command_buffer_ns": 313541.677315, "command_buffers": 2 }, { "compute_ns": 311208, "compute_span_ns": 311208, "calibration_cpu_ns": 752875, "calibration_gpu_ticks": 752875, "compute_passes": 1, "command_buffer_ns": 311249.925289, "command_buffers": 2 }, { "compute_ns": 317208, "compute_span_ns": 317208, "calibration_cpu_ns": 814750, "calibration_gpu_ticks": 814750, "compute_passes": 1, "command_buffer_ns": 317250.029184, "command_buffers": 2 }, { "compute_ns": 315583, "compute_span_ns": 315583, "calibration_cpu_ns": 702250, "calibration_gpu_ticks": 702250, "compute_passes": 1, "command_buffer_ns": 315583.311021, "command_buffers": 2 }, { "compute_ns": 319583, "compute_span_ns": 319583, "calibration_cpu_ns": 735500, "calibration_gpu_ticks": 735500, "compute_passes": 1, "command_buffer_ns": 319583.283272, "command_buffers": 2 }, { "compute_ns": 323125, "compute_span_ns": 323125, "calibration_cpu_ns": 754667, "calibration_gpu_ticks": 754667, "compute_passes": 1, "command_buffer_ns": 323166.663293, "command_buffers": 2 }, { "compute_ns": 300750, "compute_span_ns": 300750, "calibration_cpu_ns": 714792, "calibration_gpu_ticks": 714792, "compute_passes": 1, "command_buffer_ns": 300749.961752, "command_buffers": 2 }, { "compute_ns": 284917, "compute_span_ns": 284917, "calibration_cpu_ns": 674292, "calibration_gpu_ticks": 674292, "compute_passes": 1, "command_buffer_ns": 284958.339762, "command_buffers": 2 }, { "compute_ns": 312041, "compute_span_ns": 312041, "calibration_cpu_ns": 806583, "calibration_gpu_ticks": 806583, "compute_passes": 1, "command_buffer_ns": 312041.665893, "command_buffers": 2 } ], "latency": [ { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 356458, "calibration_gpu_ticks": 356458, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 457083, "calibration_gpu_ticks": 457083, "compute_passes": 1, "command_buffer_ns": 6958.26020092, "command_buffers": 2 }, { "compute_ns": 7000, "compute_span_ns": 7000, "calibration_cpu_ns": 397667, "calibration_gpu_ticks": 397667, "compute_passes": 1, "command_buffer_ns": 7083.29025656, "command_buffers": 2 }, { "compute_ns": 6916, "compute_span_ns": 6916, "calibration_cpu_ns": 351250, "calibration_gpu_ticks": 351250, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 6958, "compute_span_ns": 6958, "calibration_cpu_ns": 445416, "calibration_gpu_ticks": 445416, "compute_passes": 1, "command_buffer_ns": 6999.99509379, "command_buffers": 2 }, { "compute_ns": 9417, "compute_span_ns": 9417, "calibration_cpu_ns": 430333, "calibration_gpu_ticks": 430333, "compute_passes": 1, "command_buffer_ns": 9458.33744481, "command_buffers": 2 }, { "compute_ns": 7334, "compute_span_ns": 7334, "calibration_cpu_ns": 396208, "calibration_gpu_ticks": 396208, "compute_passes": 1, "command_buffer_ns": 7375.02705306, "command_buffers": 2 }, { "compute_ns": 10000, "compute_span_ns": 10000, "calibration_cpu_ns": 401834, "calibration_gpu_ticks": 401834, "compute_passes": 1, "command_buffer_ns": 10041.6364148, "command_buffers": 2 }, { "compute_ns": 7167, "compute_span_ns": 7167, "calibration_cpu_ns": 399375, "calibration_gpu_ticks": 399375, "compute_passes": 1, "command_buffer_ns": 7249.99699742, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 331500.021275, "command_buffers": 2 }, { "command_buffer_ns": 291666.656267, "command_buffers": 2 }, { "command_buffer_ns": 316250.021569, "command_buffers": 2 }, { "command_buffer_ns": 322124.978993, "command_buffers": 2 }, { "command_buffer_ns": 317208.352499, "command_buffers": 2 }, { "command_buffer_ns": 329333.299305, "command_buffers": 2 }, { "command_buffer_ns": 303166.685626, "command_buffers": 2 }, { "command_buffer_ns": 323124.986608, "command_buffers": 2 }, { "command_buffer_ns": 313624.972478, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 13416.6912176, "command_buffers": 2 }, { "command_buffer_ns": 7125.02514943, "command_buffers": 2 }, { "command_buffer_ns": 9500.07233769, "command_buffers": 2 }, { "command_buffer_ns": 7374.9688454, "command_buffers": 2 }, { "command_buffer_ns": 7041.67177901, "command_buffers": 2 }, { "command_buffer_ns": 6958.31840858, "command_buffers": 2 }, { "command_buffer_ns": 7208.3203122, "command_buffers": 2 }, { "command_buffer_ns": 7124.96694177, "command_buffers": 2 }, { "command_buffer_ns": 7458.26400816, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 5.179687832421875, 4.557291504171875, 4.941406587015624, 5.033202796765625, 4.956380507796874, 5.1458328016406245, 4.73697946290625, 5.0488279157500005, 4.900390194968749 ], "command_buffer_throughput_us_p50": 4.956380507796874, "command_buffer_throughput_us_p90": 5.152603807796875, "command_buffer_latency_us": [ 13.4166912176, 7.12502514943, 9.50007233769, 7.3749688454, 7.0416717790100005, 6.95831840858, 7.208320312200001, 7.1249669417699995, 7.4582640081600005 ], "command_buffer_latency_us_p50": 7.208320312200001, "command_buffer_latency_us_p90": 10.283396113672001 }, "compute_throughput_us": [ 4.899078125, 4.862625, 4.956375, 4.930984375, 4.993484375, 5.048828125, 4.69921875, 4.451828125, 4.875640625 ], "compute_throughput_us_p50": 4.899078125, "compute_throughput_us_p90": 5.004553125, "command_buffer_throughput_us": [ 4.899088708046874, 4.863280082640625, 4.9570317059999995, 4.930989234703125, 4.993488801124999, 5.049479113953125, 4.699218152375, 4.452474058781251, 4.875651029578125 ], "command_buffer_throughput_us_p50": 4.899088708046874, "command_buffer_throughput_us_p90": 5.004686863690624, "compute_latency_us": [ 7.0, 6.958, 7.0, 6.916, 6.958, 9.417, 7.334, 10.0, 7.167 ], "compute_latency_us_p50": 7.0, "compute_latency_us_p90": 9.5336, "command_buffer_latency_us": [ 6.99999509379, 6.95826020092, 7.08329025656, 6.99999509379, 6.99999509379, 9.45833744481, 7.37502705306, 10.0416364148, 7.24999699742 ], "command_buffer_latency_us_p50": 7.08329025656, "command_buffer_latency_us_p90": 9.574997238808, "counter_control_throughput_ratio": 0.9920283989350288, "counter_control_latency_ratio": 0.9765944758071007 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.85538876312, "throughput_us_p90": 4.980644206626, "latency_us_p50": 207.166, "latency_us_p90": 242.3752 }, "torch": { "cold_call_ms": 7.507833, "warmup_ms": 200.306541, "repetitions": 832, "throughput_us": [ 32.42483052884615, 30.351712740384617, 29.26307091346154, 29.647436298076926, 30.316856971153847, 29.60792307692308, 29.903496394230768, 29.372346153846156, 29.801983173076923 ], "latency_us": [ 309.042, 296.5, 310.333, 297.125, 301.16700000000003, 316.458, 306.66700000000003, 362.25, 317.83299999999997 ], "throughput_us_p50": 29.801983173076923, "throughput_us_p90": 30.766336298076922, "latency_us_p50": 309.042, "latency_us_p90": 326.71639999999996, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1192583.2950510085, "command_buffers": 7 }, { "command_buffer_ns": 1174583.3908207715, "command_buffers": 7 }, { "command_buffer_ns": 1064624.9284036458, "command_buffers": 7 }, { "command_buffer_ns": 1339583.3084359765, "command_buffers": 7 }, { "command_buffer_ns": 1314041.786827147, "command_buffers": 7 }, { "command_buffer_ns": 1346749.8938553035, "command_buffers": 7 }, { "command_buffer_ns": 1345541.6192300618, "command_buffers": 7 }, { "command_buffer_ns": 1341333.32176134, "command_buffers": 7 }, { "command_buffer_ns": 1370041.7475774884, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 89499.98300522566, "command_buffers": 1 }, { "command_buffer_ns": 88875.00734999776, "command_buffers": 1 }, { "command_buffer_ns": 70125.03920122981, "command_buffers": 1 }, { "command_buffer_ns": 93124.98150393367, "command_buffers": 1 }, { "command_buffer_ns": 88000.02979114652, "command_buffers": 1 }, { "command_buffer_ns": 104625.01086294651, "command_buffers": 1 }, { "command_buffer_ns": 103541.70808568597, "command_buffers": 1 }, { "command_buffer_ns": 89416.62963479757, "command_buffers": 1 }, { "command_buffer_ns": 88041.64826869965, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 18.634113985172007, 18.352865481574554, 16.634764506306965, 20.930989194312133, 20.531902919174172, 21.042967091489118, 21.024087800469715, 20.958333152520936, 21.406902305898257 ], "command_buffer_throughput_us_p50": 20.930989194312133, "command_buffer_throughput_us_p90": 21.115754134370945, "command_buffer_latency_us": [ 89.49998300522566, 88.87500734999776, 70.12503920122981, 93.12498150393367, 88.00002979114652, 104.62501086294651, 103.54170808568597, 89.41662963479757, 88.04164826869965 ], "command_buffer_latency_us_p50": 89.41662963479757, "command_buffer_latency_us_p90": 103.75836864113808 }, "throughput": [ { "compute_ns": 3607287.0, "compute_span_ns": 4673875.0, "command_buffer_ns": 4577083.222102374, "calibration_cpu_ns": 5505500.0, "calibration_gpu_ticks": 5505500.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3940045.0, "compute_span_ns": 5059000.0, "command_buffer_ns": 4932874.930091202, "calibration_cpu_ns": 5727625.0, "calibration_gpu_ticks": 5727625.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4017045.0, "compute_span_ns": 5116000.0, "command_buffer_ns": 5005416.751373559, "calibration_cpu_ns": 5828791.0, "calibration_gpu_ticks": 5828791.0, "compute_passes": 64, "command_buffers": 8 }, { "compute_ns": 3576166.0, "compute_span_ns": 4686667.0, "command_buffer_ns": 4579625.034239143, "calibration_cpu_ns": 5540959.0, "calibration_gpu_ticks": 5540959.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4483878.0, "compute_span_ns": 5666125.0, "command_buffer_ns": 5566541.629377753, "calibration_cpu_ns": 8416250.0, "calibration_gpu_ticks": 8416250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4190826.0, "compute_span_ns": 5395542.0, "command_buffer_ns": 5266208.318062127, "calibration_cpu_ns": 6231333.0, "calibration_gpu_ticks": 6231333.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4658835.0, "compute_span_ns": 5893000.0, "command_buffer_ns": 5774666.729848832, "calibration_cpu_ns": 6653708.0, "calibration_gpu_ticks": 6653708.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4819708.0, "compute_span_ns": 6036459.0, "command_buffer_ns": 5905166.67580232, "calibration_cpu_ns": 6843250.0, "calibration_gpu_ticks": 6843250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 5093416.0, "compute_span_ns": 6284084.0, "command_buffer_ns": 6187375.052832067, "calibration_cpu_ns": 7177875.0, "calibration_gpu_ticks": 7177875.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 89500.0, "compute_span_ns": 89500.0, "command_buffer_ns": 89499.98300522566, "calibration_cpu_ns": 618625.0, "calibration_gpu_ticks": 618625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 88875.0, "compute_span_ns": 88875.0, "command_buffer_ns": 88875.00734999776, "calibration_cpu_ns": 555375.0, "calibration_gpu_ticks": 555375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 70459.0, "compute_span_ns": 70459.0, "command_buffer_ns": 70499.95474517345, "calibration_cpu_ns": 578417.0, "calibration_gpu_ticks": 578417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 103042.0, "compute_span_ns": 103042.0, "command_buffer_ns": 103083.32275599241, "calibration_cpu_ns": 2568292.0, "calibration_gpu_ticks": 2568292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 89375.0, "compute_span_ns": 89375.0, "command_buffer_ns": 89375.01115724444, "calibration_cpu_ns": 565875.0, "calibration_gpu_ticks": 565875.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 104792.0, "compute_span_ns": 104792.0, "command_buffer_ns": 104791.65939614177, "calibration_cpu_ns": 605708.0, "calibration_gpu_ticks": 605708.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 105250.0, "compute_span_ns": 105250.0, "command_buffer_ns": 105249.98651817441, "calibration_cpu_ns": 612416.0, "calibration_gpu_ticks": 612416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 103459.0, "compute_span_ns": 103459.0, "command_buffer_ns": 103499.97319281101, "calibration_cpu_ns": 576917.0, "calibration_gpu_ticks": 576917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137000.0, "compute_span_ns": 137000.0, "command_buffer_ns": 136999.99544769526, "calibration_cpu_ns": 703416.0, "calibration_gpu_ticks": 703416.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 56.363859375, 61.563203125, 62.766328125, 55.87759375, 70.06059375, 65.48165625, 72.794296875, 75.3079375, 79.584625 ], "compute_throughput_us_p50": 65.48165625, "compute_throughput_us_p90": 76.163275, "command_buffer_throughput_us": [ 71.51692534534959, 77.07617078267504, 78.20963674021186, 71.55664115998661, 86.97721295902738, 82.28450496972073, 90.229167653888, 92.26822930941125, 96.67773520050105 ], "command_buffer_throughput_us_p50": 82.28450496972073, "command_buffer_throughput_us_p90": 93.15013048762921, "compute_latency_us": [ 89.5, 88.875, 70.459, 103.042, 89.375, 104.792, 105.25, 103.459, 137.0 ], "compute_latency_us_p50": 103.042, "compute_latency_us_p90": 111.60000000000001, "command_buffer_latency_us": [ 89.49998300522566, 88.87500734999776, 70.49995474517345, 103.08332275599241, 89.37501115724444, 104.79165939614177, 105.24998651817441, 103.49997319281101, 136.99999544769526 ], "command_buffer_latency_us_p50": 103.08332275599241, "command_buffer_latency_us_p90": 111.59998830407858, "counter_control_throughput_ratio": 4.236198334924026, "counter_control_latency_ratio": 1.015624782961565 }, "download_ms": 0.299666, "allocation_upload_ms": 0.682583, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.16292166648514694, "device_compute_slowdown": 0.07481603865204615, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 37, "n": 1537, "k": 1 }, "name": "softmax_37x1537", "backend": "metal", "block": [ 1, 1537, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.563708, "warmup_ms": 200.454708, "repetitions": 880, "throughput_us": [ 29.753361363636362, 29.053503409090908, 29.437073863636364, 29.496401136363634, 29.49715909090909, 29.2725375, 29.26998068181818, 29.29962159090909, 29.204877272727273 ], "latency_us": [ 327.0, 299.833, 310.333, 313.58399999999995, 292.292, 309.125, 307.70799999999997, 298.79200000000003, 332.792 ], "throughput_us_p50": 29.29962159090909, "throughput_us_p90": 29.548399545454544, "latency_us_p50": 309.125, "latency_us_p90": 328.1584, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1062624.97138232, "command_buffers": 7 }, { "command_buffer_ns": 1064333.3080224693, "command_buffers": 7 }, { "command_buffer_ns": 1074541.7093858123, "command_buffers": 7 }, { "command_buffer_ns": 1061791.7287163436, "command_buffers": 7 }, { "command_buffer_ns": 1104583.265259862, "command_buffers": 7 }, { "command_buffer_ns": 1060541.660990566, "command_buffers": 7 }, { "command_buffer_ns": 1217458.3389423788, "command_buffers": 7 }, { "command_buffer_ns": 1214666.641317308, "command_buffers": 7 }, { "command_buffer_ns": 1063541.6838340461, "command_buffers": 7 } ], "latency": [ { "command_buffer_ns": 62583.30540731549, "command_buffers": 1 }, { "command_buffer_ns": 71500.02056732774, "command_buffers": 1 }, { "command_buffer_ns": 76125.02668052912, "command_buffers": 1 }, { "command_buffer_ns": 74749.98710677028, "command_buffers": 1 }, { "command_buffer_ns": 58458.36130902171, "command_buffers": 1 }, { "command_buffer_ns": 70750.01485645771, "command_buffers": 1 }, { "command_buffer_ns": 75791.67140647769, "command_buffers": 1 }, { "command_buffer_ns": 72875.00193342566, "command_buffers": 1 }, { "command_buffer_ns": 76500.00043213367, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 16.60351517784875, 16.630207937851083, 16.789714209153317, 16.59049576119287, 17.259113519685343, 16.570963452977594, 19.022786545974668, 18.979166270582937, 16.61783880990697 ], "command_buffer_throughput_us_p50": 16.630207937851083, "command_buffer_throughput_us_p90": 18.987890325661283, "command_buffer_latency_us": [ 62.58330540731549, 71.50002056732774, 76.12502668052912, 74.74998710677028, 58.45836130902171, 70.75001485645771, 75.79167140647769, 72.87500193342566, 76.50000043213367 ], "command_buffer_latency_us_p50": 72.87500193342566, "command_buffer_latency_us_p90": 76.20002143085003 }, "throughput": [ { "compute_ns": 3482877.0, "compute_span_ns": 4522458.0, "command_buffer_ns": 4414041.701238602, "calibration_cpu_ns": 5237666.0, "calibration_gpu_ticks": 5237666.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3475879.0, "compute_span_ns": 4564000.0, "command_buffer_ns": 4464083.234779537, "calibration_cpu_ns": 5317250.0, "calibration_gpu_ticks": 5317250.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3520330.0, "compute_span_ns": 4448916.0, "command_buffer_ns": 4379791.673272848, "calibration_cpu_ns": 5119666.0, "calibration_gpu_ticks": 5119666.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3738581.0, "compute_span_ns": 4820750.0, "command_buffer_ns": 4727750.027086586, "calibration_cpu_ns": 5496500.0, "calibration_gpu_ticks": 5496500.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3696453.0, "compute_span_ns": 4849833.0, "command_buffer_ns": 4731833.2944996655, "calibration_cpu_ns": 5521916.0, "calibration_gpu_ticks": 5521916.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3920833.0, "compute_span_ns": 5001167.0, "command_buffer_ns": 4903833.323623985, "calibration_cpu_ns": 5689959.0, "calibration_gpu_ticks": 5689959.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3513374.0, "compute_span_ns": 4621917.0, "command_buffer_ns": 4458833.253011107, "calibration_cpu_ns": 5555042.0, "calibration_gpu_ticks": 5555042.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 3826292.0, "compute_span_ns": 4984834.0, "command_buffer_ns": 4876291.612163186, "calibration_cpu_ns": 5797833.0, "calibration_gpu_ticks": 5797833.0, "compute_passes": 64, "command_buffers": 7 }, { "compute_ns": 4423457.0, "compute_span_ns": 5552833.0, "command_buffer_ns": 5446708.470117301, "calibration_cpu_ns": 6247167.0, "calibration_gpu_ticks": 6247167.0, "compute_passes": 64, "command_buffers": 7 } ], "latency": [ { "compute_ns": 57875.0, "compute_span_ns": 57875.0, "command_buffer_ns": 57874.94592368603, "calibration_cpu_ns": 531917.0, "calibration_gpu_ticks": 531917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 62667.0, "compute_span_ns": 62667.0, "command_buffer_ns": 62666.65877774358, "calibration_cpu_ns": 533167.0, "calibration_gpu_ticks": 533167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 76625.0, "compute_span_ns": 76625.0, "command_buffer_ns": 76624.97228011489, "calibration_cpu_ns": 532500.0, "calibration_gpu_ticks": 532500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 77333.0, "compute_span_ns": 77333.0, "command_buffer_ns": 77333.30130577087, "calibration_cpu_ns": 531333.0, "calibration_gpu_ticks": 531333.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 56875.0, "compute_span_ns": 56875.0, "command_buffer_ns": 56874.99651685357, "calibration_cpu_ns": 473000.0, "calibration_gpu_ticks": 473000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 63417.0, "compute_span_ns": 63417.0, "command_buffer_ns": 63416.664488613605, "calibration_cpu_ns": 508583.0, "calibration_gpu_ticks": 508583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 76416.0, "compute_span_ns": 76416.0, "command_buffer_ns": 76374.97037649155, "calibration_cpu_ns": 550416.0, "calibration_gpu_ticks": 550416.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 78458.0, "compute_span_ns": 78458.0, "command_buffer_ns": 78458.28076824546, "calibration_cpu_ns": 539000.0, "calibration_gpu_ticks": 539000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 74333.0, "compute_span_ns": 74333.0, "command_buffer_ns": 74291.65998473763, "calibration_cpu_ns": 520500.0, "calibration_gpu_ticks": 520500.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 54.419953125, 54.310609375, 55.00515625, 58.415328125, 57.757078125, 61.263015625, 54.89646875, 59.7858125, 69.116515625 ], "compute_throughput_us_p50": 57.757078125, "compute_throughput_us_p90": 62.833715625, "command_buffer_throughput_us": [ 68.96940158185316, 69.75130054343026, 68.43424489488825, 73.8710941732279, 73.93489522655727, 76.62239568162477, 69.66926957829855, 76.19205644004978, 85.10481984558282 ], "command_buffer_throughput_us_p50": 73.8710941732279, "command_buffer_throughput_us_p90": 78.31888051441638, "compute_latency_us": [ 57.875, 62.667, 76.625, 77.333, 56.875, 63.417, 76.416, 78.458, 74.333 ], "compute_latency_us_p50": 74.333, "compute_latency_us_p90": 77.55799999999999, "command_buffer_latency_us": [ 57.87494592368603, 62.66665877774358, 76.62497228011489, 77.33330130577087, 56.87499651685357, 63.416664488613605, 76.37497037649155, 78.45828076824546, 74.29165998473763 ], "command_buffer_latency_us_p50": 74.29165998473763, "command_buffer_latency_us_p90": 77.55829719826579, "counter_control_throughput_ratio": 4.194253060701258, "counter_control_latency_ratio": 0.9729146565741352 }, "download_ms": 0.281833, "allocation_upload_ms": 0.711125, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 4.2122453832241225e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "37", "1537", "1", "1", "1537", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-4ti1myrn/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "1ca0ba2a59ee9b200a0f7933e3c823dd873803c26a844f54bebdc043f01bc250", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.376417, "capture_ms": 0.0685, "compile_ms": 49.153916, "allocation_upload_ms": 1.255417, "cold_call_ms": 1.855125, "warmup_ms": 200.28775, "download_ms": 0.309125, "repetitions": 5566, "output_elements": 56869, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 4.78623194395, 3.98021469637, 3.95511444484, 4.07747933884, 4.73653287819, 4.76630434783, 4.52477093065, 3.97143370464, 3.98489346029 ], "latency_us": [ 312.417, 216.083, 241.333, 270.083, 185.5, 190.833, 208.5, 186.75, 188.542 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 37, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 37, "reduction_scalar_rounds": 40, "reduction_lane_utilization": 0.75048828125, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 6148, "global_write_bytes": 6148, "private_read_bytes": 36888, "private_write_bytes": 12296 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 32, "global_write_bytes": 32, "private_read_bytes": 192, "private_write_bytes": 64 }, "striped_storage_scalars_per_worker": 16, "reduction_operations": 2, "reduction_elements": 3074, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 88, "normalized_kernel_cost": 88, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 4611, "fragment_scalars_per_lane": 0, "concurrent_waves": 1, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 261459, "compute_span_ns": 261459, "calibration_cpu_ns": 813541, "calibration_gpu_ticks": 813541, "compute_passes": 1, "command_buffer_ns": 261583.307292, "command_buffers": 2 }, { "compute_ns": 243250, "compute_span_ns": 243250, "calibration_cpu_ns": 707208, "calibration_gpu_ticks": 707208, "compute_passes": 1, "command_buffer_ns": 243333.284743, "command_buffers": 2 }, { "compute_ns": 262250, "compute_span_ns": 262250, "calibration_cpu_ns": 759958, "calibration_gpu_ticks": 759958, "compute_passes": 1, "command_buffer_ns": 262374.989688, "command_buffers": 2 }, { "compute_ns": 281500, "compute_span_ns": 281500, "calibration_cpu_ns": 789417, "calibration_gpu_ticks": 789417, "compute_passes": 1, "command_buffer_ns": 281624.961644, "command_buffers": 2 }, { "compute_ns": 261750, "compute_span_ns": 261750, "calibration_cpu_ns": 685667, "calibration_gpu_ticks": 685667, "compute_passes": 1, "command_buffer_ns": 261791.690718, "command_buffers": 2 }, { "compute_ns": 261625, "compute_span_ns": 261625, "calibration_cpu_ns": 683084, "calibration_gpu_ticks": 683084, "compute_passes": 1, "command_buffer_ns": 261666.71887, "command_buffers": 2 }, { "compute_ns": 243959, "compute_span_ns": 243959, "calibration_cpu_ns": 703291, "calibration_gpu_ticks": 703291, "compute_passes": 1, "command_buffer_ns": 243999.995291, "command_buffers": 2 }, { "compute_ns": 261750, "compute_span_ns": 261750, "calibration_cpu_ns": 681041, "calibration_gpu_ticks": 681041, "compute_passes": 1, "command_buffer_ns": 261916.662566, "command_buffers": 2 }, { "compute_ns": 262416, "compute_span_ns": 262416, "calibration_cpu_ns": 704375, "calibration_gpu_ticks": 704375, "compute_passes": 1, "command_buffer_ns": 262458.343059, "command_buffers": 2 } ], "latency": [ { "compute_ns": 6000, "compute_span_ns": 6000, "calibration_cpu_ns": 386500, "calibration_gpu_ticks": 386500, "compute_passes": 1, "command_buffer_ns": 6083.39905739, "command_buffers": 2 }, { "compute_ns": 6500, "compute_span_ns": 6500, "calibration_cpu_ns": 389416, "calibration_gpu_ticks": 389416, "compute_passes": 1, "command_buffer_ns": 6583.28644931, "command_buffers": 2 }, { "compute_ns": 6375, "compute_span_ns": 6375, "calibration_cpu_ns": 553750, "calibration_gpu_ticks": 553750, "compute_passes": 1, "command_buffer_ns": 6458.31460133, "command_buffers": 2 }, { "compute_ns": 6459, "compute_span_ns": 6459, "calibration_cpu_ns": 415667, "calibration_gpu_ticks": 415667, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "compute_ns": 6459, "compute_span_ns": 6459, "calibration_cpu_ns": 391333, "calibration_gpu_ticks": 391333, "compute_passes": 1, "command_buffer_ns": 6625.02134219, "command_buffers": 2 }, { "compute_ns": 6417, "compute_span_ns": 6417, "calibration_cpu_ns": 410000, "calibration_gpu_ticks": 410000, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "compute_ns": 6458, "compute_span_ns": 6458, "calibration_cpu_ns": 401833, "calibration_gpu_ticks": 401833, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 }, { "compute_ns": 6459, "compute_span_ns": 6459, "calibration_cpu_ns": 401250, "calibration_gpu_ticks": 401250, "compute_passes": 1, "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "compute_ns": 6458, "compute_span_ns": 6458, "calibration_cpu_ns": 404333, "calibration_gpu_ticks": 404333, "compute_passes": 1, "command_buffer_ns": 6500.04949421, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 265458.307695, "command_buffers": 2 }, { "command_buffer_ns": 242625.013925, "command_buffers": 2 }, { "command_buffer_ns": 261708.337348, "command_buffers": 2 }, { "command_buffer_ns": 243375.019636, "command_buffers": 2 }, { "command_buffer_ns": 242875.015829, "command_buffers": 2 }, { "command_buffer_ns": 262625.0498, "command_buffers": 2 }, { "command_buffer_ns": 244125.025347, "command_buffers": 2 }, { "command_buffer_ns": 261500.01213, "command_buffers": 2 }, { "command_buffer_ns": 262000.015937, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 6083.39905739, "command_buffers": 2 }, { "command_buffer_ns": 6625.02134219, "command_buffers": 2 }, { "command_buffer_ns": 6541.66797176, "command_buffers": 2 }, { "command_buffer_ns": 6375.01943856, "command_buffers": 2 }, { "command_buffer_ns": 6583.34465697, "command_buffers": 2 }, { "command_buffer_ns": 7083.34846422, "command_buffers": 2 }, { "command_buffer_ns": 6499.99128655, "command_buffers": 2 }, { "command_buffer_ns": 6458.31460133, "command_buffers": 2 }, { "command_buffer_ns": 6541.66797176, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 4.1477860577343755, 3.7910158425781253, 4.0891927710625, 3.8027346818125003, 3.794922122328125, 4.103516403125, 3.814453521046875, 4.08593768953125, 4.093750249015625 ], "command_buffer_throughput_us_p50": 4.08593768953125, "command_buffer_throughput_us_p90": 4.112370334046875, "command_buffer_latency_us": [ 6.08339905739, 6.62502134219, 6.54166797176, 6.37501943856, 6.58334465697, 7.08334846422, 6.49999128655, 6.458314601330001, 6.54166797176 ], "command_buffer_latency_us_p50": 6.54166797176, "command_buffer_latency_us_p90": 6.716686766596 }, "compute_throughput_us": [ 4.085296875, 3.80078125, 4.09765625, 4.3984375, 4.08984375, 4.087890625, 3.811859375, 4.08984375, 4.10025 ], "compute_throughput_us_p50": 4.08984375, "compute_throughput_us_p90": 4.1598875, "command_buffer_throughput_us": [ 4.0872391764375005, 3.802082574109375, 4.099609213875, 4.400390025687501, 4.09049516746875, 4.08854248234375, 3.812499926421875, 4.09244785259375, 4.100911610296874 ], "command_buffer_throughput_us_p50": 4.09049516746875, "command_buffer_throughput_us_p90": 4.160807293375, "compute_latency_us": [ 6.0, 6.5, 6.375, 6.459, 6.459, 6.417, 6.458, 6.459, 6.458 ], "compute_latency_us_p50": 6.458, "compute_latency_us_p90": 6.4672, "command_buffer_latency_us": [ 6.08339905739, 6.58328644931, 6.458314601330001, 6.500049494210001, 6.62502134219, 6.500049494210001, 6.500049494210001, 6.49999128655, 6.500049494210001 ], "command_buffer_latency_us_p50": 6.500049494210001, "command_buffer_latency_us_p90": 6.591633427886, "counter_control_throughput_ratio": 1.001749340053896, "counter_control_latency_ratio": 1.0 }, "correctness": { "max_abs_error": 2.6856349761260334e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 4.07747933884, "throughput_us_p90": 4.770289867054, "latency_us_p50": 208.5, "latency_us_p90": 278.5498 }, "slowdown": 0.13916491467948294, "device_compute_slowdown": 0.07081112623371648, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-72qcqc8b/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "0d8a82e58cbe97c43d15ef559cf043ad575745e0d3fb0f4f40ed60b2caca21c0", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.096375, "capture_ms": 0.070958, "compile_ms": 49.412584, "allocation_upload_ms": 1.847041, "cold_call_ms": 5.445458, "warmup_ms": 200.025292, "download_ms": 0.905875, "repetitions": 1518, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 13.8873247694, 14.2367147563, 14.2653162055, 13.6109466403, 14.3277068511, 14.6312312253, 13.9863306983, 13.5355184453, 14.3244400527 ], "latency_us": [ 268.334, 276.5, 246.958, 245.708, 284.792, 257.75, 254.917, 251.25, 263.166 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 256, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 108, "normalized_kernel_cost": 432, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 899583, "compute_span_ns": 899583, "calibration_cpu_ns": 1367625, "calibration_gpu_ticks": 1367625, "compute_passes": 1, "command_buffer_ns": 899583.334103, "command_buffers": 2 }, { "compute_ns": 890667, "compute_span_ns": 890667, "calibration_cpu_ns": 1327833, "calibration_gpu_ticks": 1327833, "compute_passes": 1, "command_buffer_ns": 890833.325684, "command_buffers": 2 }, { "compute_ns": 924542, "compute_span_ns": 924542, "calibration_cpu_ns": 1387708, "calibration_gpu_ticks": 1387708, "compute_passes": 1, "command_buffer_ns": 924624.96832, "command_buffers": 2 }, { "compute_ns": 861167, "compute_span_ns": 861167, "calibration_cpu_ns": 1287709, "calibration_gpu_ticks": 1287709, "compute_passes": 1, "command_buffer_ns": 861291.71541, "command_buffers": 2 }, { "compute_ns": 884375, "compute_span_ns": 884375, "calibration_cpu_ns": 1356208, "calibration_gpu_ticks": 1356208, "compute_passes": 1, "command_buffer_ns": 884416.62956, "command_buffers": 2 }, { "compute_ns": 879041, "compute_span_ns": 879041, "calibration_cpu_ns": 1336084, "calibration_gpu_ticks": 1336084, "compute_passes": 1, "command_buffer_ns": 879083.352629, "command_buffers": 2 }, { "compute_ns": 918334, "compute_span_ns": 918334, "calibration_cpu_ns": 1312083, "calibration_gpu_ticks": 1312083, "compute_passes": 1, "command_buffer_ns": 918416.655622, "command_buffers": 2 }, { "compute_ns": 893916, "compute_span_ns": 893916, "calibration_cpu_ns": 1352417, "calibration_gpu_ticks": 1352417, "compute_passes": 1, "command_buffer_ns": 893999.997061, "command_buffers": 2 }, { "compute_ns": 882834, "compute_span_ns": 882834, "calibration_cpu_ns": 1290375, "calibration_gpu_ticks": 1290375, "compute_passes": 1, "command_buffer_ns": 882999.971509, "command_buffers": 2 } ], "latency": [ { "compute_ns": 15709, "compute_span_ns": 15709, "calibration_cpu_ns": 401208, "calibration_gpu_ticks": 401208, "compute_passes": 1, "command_buffer_ns": 15750.0035129, "command_buffers": 2 }, { "compute_ns": 18250, "compute_span_ns": 18250, "calibration_cpu_ns": 444250, "calibration_gpu_ticks": 444250, "compute_passes": 1, "command_buffer_ns": 18333.2595043, "command_buffers": 2 }, { "compute_ns": 18375, "compute_span_ns": 18375, "calibration_cpu_ns": 446667, "calibration_gpu_ticks": 446667, "compute_passes": 1, "command_buffer_ns": 18416.72929, "command_buffers": 2 }, { "compute_ns": 18292, "compute_span_ns": 18292, "calibration_cpu_ns": 423000, "calibration_gpu_ticks": 423000, "compute_passes": 1, "command_buffer_ns": 18375.0526048, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 508750, "calibration_gpu_ticks": 508750, "compute_passes": 1, "command_buffer_ns": 18583.3778232, "command_buffers": 2 }, { "compute_ns": 17917, "compute_span_ns": 17917, "calibration_cpu_ns": 426959, "calibration_gpu_ticks": 426959, "compute_passes": 1, "command_buffer_ns": 17999.9624379, "command_buffers": 2 }, { "compute_ns": 18250, "compute_span_ns": 18250, "calibration_cpu_ns": 424375, "calibration_gpu_ticks": 424375, "compute_passes": 1, "command_buffer_ns": 18375.0526048, "command_buffers": 2 }, { "compute_ns": 18541, "compute_span_ns": 18541, "calibration_cpu_ns": 429458, "calibration_gpu_ticks": 429458, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "compute_ns": 18167, "compute_span_ns": 18167, "calibration_cpu_ns": 492792, "calibration_gpu_ticks": 492792, "compute_passes": 1, "command_buffer_ns": 18166.6691788, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 950458.343141, "command_buffers": 2 }, { "command_buffer_ns": 986124.97095, "command_buffers": 2 }, { "command_buffer_ns": 926833.308768, "command_buffers": 2 }, { "command_buffer_ns": 931749.99347, "command_buffers": 2 }, { "command_buffer_ns": 879750.00497, "command_buffers": 2 }, { "command_buffer_ns": 943000.020925, "command_buffers": 2 }, { "command_buffer_ns": 910791.684873, "command_buffers": 2 }, { "command_buffer_ns": 957250.071224, "command_buffers": 2 }, { "command_buffer_ns": 908958.318178, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 15583.3549798, "command_buffers": 2 }, { "command_buffer_ns": 18458.2895599, "command_buffers": 2 }, { "command_buffer_ns": 18416.6710824, "command_buffers": 2 }, { "command_buffer_ns": 18499.9662451, "command_buffers": 2 }, { "command_buffer_ns": 21125.015337, "command_buffers": 2 }, { "command_buffer_ns": 18083.3158083, "command_buffers": 2 }, { "command_buffer_ns": 18208.345864, "command_buffers": 2 }, { "command_buffer_ns": 18291.6410267, "command_buffers": 2 }, { "command_buffer_ns": 18333.3759196, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 14.850911611578125, 15.40820267109375, 14.481770449499999, 14.558593647968749, 13.74609382765625, 14.734375326953124, 14.231120076140625, 14.957032362875, 14.20247372153125 ], "command_buffer_throughput_us_p50": 14.558593647968749, "command_buffer_throughput_us_p90": 15.04726642451875, "command_buffer_latency_us": [ 15.583354979800001, 18.4582895599, 18.4166710824, 18.4999662451, 21.125015337, 18.0833158083, 18.208345863999998, 18.2916410267, 18.333375919599998 ], "command_buffer_latency_us_p50": 18.333375919599998, "command_buffer_latency_us_p90": 19.02497606348 }, "compute_throughput_us": [ 14.055984375, 13.916671875, 14.44596875, 13.455734375, 13.818359375, 13.735015625, 14.34896875, 13.9674375, 13.79428125 ], "compute_throughput_us_p50": 13.916671875, "compute_throughput_us_p90": 14.36836875, "command_buffer_throughput_us": [ 14.055989595359375, 13.9192707138125, 14.44726513, 13.457683053281249, 13.819009836875, 13.735677384828124, 14.35026024409375, 13.968749954078124, 13.796874554828126 ], "command_buffer_throughput_us_p50": 13.9192707138125, "command_buffer_throughput_us_p90": 14.369661221275, "compute_latency_us": [ 15.709, 18.25, 18.375, 18.292, 18.5, 17.917, 18.25, 18.541, 18.167 ], "compute_latency_us_p50": 18.25, "compute_latency_us_p90": 18.5082, "command_buffer_latency_us": [ 15.750003512900001, 18.3332595043, 18.41672929, 18.3750526048, 18.5833778232, 17.999962437900002, 18.3750526048, 18.541642930400002, 18.1666691788 ], "command_buffer_latency_us_p50": 18.3750526048, "command_buffer_latency_us_p90": 18.54998990896, "counter_control_throughput_ratio": 0.9464731838011203, "counter_control_latency_ratio": 0.9953905925614736 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 14.2367147563, "throughput_us_p90": 14.388411725940001, "latency_us_p50": 257.75, "latency_us_p90": 278.1584 }, "torch": { "cold_call_ms": 6.833083, "warmup_ms": 200.488042, "repetitions": 812, "throughput_us": [ 33.462182266009854, 33.57953694581281, 33.50307881773399, 33.37843842364532, 35.06291009852217, 33.71177093596059, 33.23286083743842, 33.16666748768473, 33.44227216748769 ], "latency_us": [ 312.042, 277.95799999999997, 278.916, 292.208, 276.625, 279.667, 282.58299999999997, 294.75, 273.084 ], "throughput_us_p50": 33.462182266009854, "throughput_us_p90": 33.981998768472906, "latency_us_p50": 279.667, "latency_us_p90": 298.2084, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1868250.0813156366, "command_buffers": 11 }, { "command_buffer_ns": 1879333.40203017, "command_buffers": 11 }, { "command_buffer_ns": 1824749.9829158187, "command_buffers": 11 }, { "command_buffer_ns": 1831666.6828468442, "command_buffers": 11 }, { "command_buffer_ns": 1820666.8901257217, "command_buffers": 11 }, { "command_buffer_ns": 1851374.9819248915, "command_buffers": 11 }, { "command_buffer_ns": 1830374.8802281916, "command_buffers": 11 }, { "command_buffer_ns": 1835583.4763497114, "command_buffers": 11 }, { "command_buffer_ns": 1824791.6596010327, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 46458.328142762184, "command_buffers": 1 }, { "command_buffer_ns": 47874.986194074154, "command_buffers": 1 }, { "command_buffer_ns": 48291.69483855367, "command_buffers": 1 }, { "command_buffer_ns": 48250.018153339624, "command_buffers": 1 }, { "command_buffer_ns": 47874.986194074154, "command_buffers": 1 }, { "command_buffer_ns": 47624.98429045081, "command_buffers": 1 }, { "command_buffer_ns": 47666.660975664854, "command_buffers": 1 }, { "command_buffer_ns": 47708.3376608789, "command_buffers": 1 }, { "command_buffer_ns": 47750.01434609294, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.191407520556822, 29.364584406721406, 28.511718483059667, 28.61979191948194, 28.4479201582144, 28.92773409257643, 28.599607503565494, 28.68099181796424, 28.512369681266136 ], "command_buffer_throughput_us_p50": 28.61979191948194, "command_buffer_throughput_us_p90": 29.22604289778974, "command_buffer_latency_us": [ 46.458328142762184, 47.874986194074154, 48.29169483855367, 48.250018153339624, 47.874986194074154, 47.62498429045081, 47.666660975664854, 47.7083376608789, 47.75001434609294 ], "command_buffer_latency_us_p50": 47.75001434609294, "command_buffer_latency_us_p90": 48.25835349038243 }, "throughput": [ { "compute_ns": 3017500.0, "compute_span_ns": 4019875.0, "command_buffer_ns": 3831708.221696317, "calibration_cpu_ns": 4639625.0, "calibration_gpu_ticks": 4639625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3107375.0, "compute_span_ns": 3919250.0, "command_buffer_ns": 3786083.310842514, "calibration_cpu_ns": 4550000.0, "calibration_gpu_ticks": 4550000.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2989836.0, "compute_span_ns": 3880250.0, "command_buffer_ns": 3737249.9937191606, "calibration_cpu_ns": 4561708.0, "calibration_gpu_ticks": 4561708.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2978620.0, "compute_span_ns": 3866792.0, "command_buffer_ns": 3724833.3101160824, "calibration_cpu_ns": 4572917.0, "calibration_gpu_ticks": 4572917.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2993585.0, "compute_span_ns": 3949083.0, "command_buffer_ns": 3819624.9515749514, "calibration_cpu_ns": 4623875.0, "calibration_gpu_ticks": 4623875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2985003.0, "compute_span_ns": 4101834.0, "command_buffer_ns": 3976499.95284155, "calibration_cpu_ns": 4801375.0, "calibration_gpu_ticks": 4801375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2990624.0, "compute_span_ns": 3892042.0, "command_buffer_ns": 3726791.706867516, "calibration_cpu_ns": 4573625.0, "calibration_gpu_ticks": 4573625.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2981289.0, "compute_span_ns": 3846542.0, "command_buffer_ns": 3693916.7184755206, "calibration_cpu_ns": 4600209.0, "calibration_gpu_ticks": 4600209.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2994085.0, "compute_span_ns": 3907792.0, "command_buffer_ns": 3745458.205230534, "calibration_cpu_ns": 4591792.0, "calibration_gpu_ticks": 4591792.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 47667.0, "compute_span_ns": 47667.0, "command_buffer_ns": 47666.660975664854, "calibration_cpu_ns": 558791.0, "calibration_gpu_ticks": 558791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47625.0, "compute_span_ns": 47625.0, "command_buffer_ns": 47624.98429045081, "calibration_cpu_ns": 493584.0, "calibration_gpu_ticks": 493584.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47917.0, "compute_span_ns": 47917.0, "command_buffer_ns": 47958.33956450224, "calibration_cpu_ns": 538833.0, "calibration_gpu_ticks": 538833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47542.0, "compute_span_ns": 47542.0, "command_buffer_ns": 47541.68912768364, "calibration_cpu_ns": 487833.0, "calibration_gpu_ticks": 487833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 48250.0, "compute_span_ns": 48250.0, "command_buffer_ns": 48250.018153339624, "calibration_cpu_ns": 580541.0, "calibration_gpu_ticks": 580541.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47916.0, "compute_span_ns": 47916.0, "command_buffer_ns": 47874.986194074154, "calibration_cpu_ns": 508750.0, "calibration_gpu_ticks": 508750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47917.0, "compute_span_ns": 47917.0, "command_buffer_ns": 47958.33956450224, "calibration_cpu_ns": 568375.0, "calibration_gpu_ticks": 568375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47750.0, "compute_span_ns": 47750.0, "command_buffer_ns": 47749.956138432026, "calibration_cpu_ns": 514833.0, "calibration_gpu_ticks": 514833.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 48125.0, "compute_span_ns": 48125.0, "command_buffer_ns": 48124.9880976975, "calibration_cpu_ns": 601125.0, "calibration_gpu_ticks": 601125.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.1484375, 48.552734375, 46.7161875, 46.5409375, 46.774765625, 46.640671875, 46.7285, 46.582640625, 46.782578125 ], "compute_throughput_us_p50": 46.7285, "compute_throughput_us_p90": 47.429296875, "command_buffer_throughput_us": [ 59.870440964004956, 59.15755173191428, 58.394531151861884, 58.20052047056379, 59.681639868358616, 62.13281176314922, 58.23112041980494, 57.71744872618001, 58.522784456727095 ], "command_buffer_throughput_us_p50": 58.522784456727095, "command_buffer_throughput_us_p90": 60.32291512383381, "compute_latency_us": [ 47.667, 47.625, 47.917, 47.542, 48.25, 47.916, 47.917, 47.75, 48.125 ], "compute_latency_us_p50": 47.916, "compute_latency_us_p90": 48.15, "command_buffer_latency_us": [ 47.666660975664854, 47.62498429045081, 47.95833956450224, 47.54168912768364, 48.250018153339624, 47.874986194074154, 47.95833956450224, 47.749956138432026, 48.124988097697496 ], "command_buffer_latency_us_p50": 47.874986194074154, "command_buffer_latency_us_p90": 48.14999410882592, "counter_control_throughput_ratio": 2.048088795017992, "counter_control_latency_ratio": 1.0052493855346734 }, "download_ms": 0.35975, "allocation_upload_ms": 1.091875, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.4254568528473213, "device_compute_slowdown": 0.29781978610483967, "valid": true, "round": 3, "variant": "reference" }, { "case": { "operation": "softmax", "m": 256, "n": 3072, "k": 1 }, "name": "softmax_256x3072", "backend": "metal", "block": [ 1, 3072, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.550875, "warmup_ms": 200.003292, "repetitions": 630, "throughput_us": [ 42.45615079365079, 33.654695238095236, 33.79358412698412, 33.505290476190474, 33.494509523809526, 33.4365746031746, 33.51488095238095, 37.88439206349206, 33.851257142857136 ], "latency_us": [ 318.20799999999997, 413.208, 314.167, 297.08299999999997, 296.916, 440.70799999999997, 296.125, 310.66700000000003, 273.666 ], "throughput_us_p50": 33.654695238095236, "throughput_us_p90": 38.798743809523806, "latency_us_p50": 310.66700000000003, "latency_us_p90": 418.708, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 1892208.4709629416, "command_buffers": 11 }, { "command_buffer_ns": 1859083.2478366792, "command_buffers": 11 }, { "command_buffer_ns": 1825416.6352562606, "command_buffers": 11 }, { "command_buffer_ns": 1829291.5192432702, "command_buffers": 11 }, { "command_buffer_ns": 1817416.7489632964, "command_buffers": 11 }, { "command_buffer_ns": 1826374.907977879, "command_buffers": 11 }, { "command_buffer_ns": 1827208.383474499, "command_buffers": 11 }, { "command_buffer_ns": 1830208.3481103182, "command_buffers": 11 }, { "command_buffer_ns": 1850124.9724067748, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 48874.993808567524, "command_buffers": 1 }, { "command_buffer_ns": 47416.65907204151, "command_buffers": 1 }, { "command_buffer_ns": 48666.668590158224, "command_buffers": 1 }, { "command_buffer_ns": 47833.367716521025, "command_buffers": 1 }, { "command_buffer_ns": 51333.33615958691, "command_buffers": 1 }, { "command_buffer_ns": 49624.99951943755, "command_buffers": 1 }, { "command_buffer_ns": 48749.963752925396, "command_buffers": 1 }, { "command_buffer_ns": 47374.98238682747, "command_buffers": 1 }, { "command_buffer_ns": 48249.95994567871, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 29.565757358795963, 29.048175747448113, 28.522134925879072, 28.582679988176096, 28.397136702551506, 28.53710793715436, 28.550130991789047, 28.59700543922372, 28.908202693855856 ], "command_buffer_throughput_us_p50": 28.582679988176096, "command_buffer_throughput_us_p90": 29.151692069717683, "command_buffer_latency_us": [ 48.874993808567524, 47.41665907204151, 48.666668590158224, 47.833367716521025, 51.333336159586906, 49.62499951943755, 48.749963752925396, 47.37498238682747, 48.24995994567871 ], "command_buffer_latency_us_p50": 48.666668590158224, "command_buffer_latency_us_p90": 49.96666684746742 }, "throughput": [ { "compute_ns": 3026755.0, "compute_span_ns": 3954250.0, "command_buffer_ns": 3806750.0572651625, "calibration_cpu_ns": 4799375.0, "calibration_gpu_ticks": 4799375.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 2969373.0, "compute_span_ns": 3883666.0, "command_buffer_ns": 3706583.345774561, "calibration_cpu_ns": 4547167.0, "calibration_gpu_ticks": 4547167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3024752.0, "compute_span_ns": 3995458.0, "command_buffer_ns": 3817375.109065324, "calibration_cpu_ns": 4627042.0, "calibration_gpu_ticks": 4627042.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3037080.0, "compute_span_ns": 3923833.0, "command_buffer_ns": 3785375.156439841, "calibration_cpu_ns": 4541667.0, "calibration_gpu_ticks": 4541667.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3022798.0, "compute_span_ns": 4038875.0, "command_buffer_ns": 3866249.9864585698, "calibration_cpu_ns": 4820667.0, "calibration_gpu_ticks": 4820667.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3017253.0, "compute_span_ns": 3969083.0, "command_buffer_ns": 3809541.6966825724, "calibration_cpu_ns": 4709500.0, "calibration_gpu_ticks": 4709500.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3014622.0, "compute_span_ns": 4034250.0, "command_buffer_ns": 3856624.9422729015, "calibration_cpu_ns": 4673916.0, "calibration_gpu_ticks": 4673916.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3048335.0, "compute_span_ns": 4034792.0, "command_buffer_ns": 3873125.009704381, "calibration_cpu_ns": 4735666.0, "calibration_gpu_ticks": 4735666.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 3020214.0, "compute_span_ns": 3972292.0, "command_buffer_ns": 3816374.985035509, "calibration_cpu_ns": 4662125.0, "calibration_gpu_ticks": 4662125.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 47083.0, "compute_span_ns": 47083.0, "command_buffer_ns": 47083.303797990084, "calibration_cpu_ns": 502208.0, "calibration_gpu_ticks": 502208.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49917.0, "compute_span_ns": 49917.0, "command_buffer_ns": 49958.296585828066, "calibration_cpu_ns": 565791.0, "calibration_gpu_ticks": 565791.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 52209.0, "compute_span_ns": 52209.0, "command_buffer_ns": 52249.99040365219, "calibration_cpu_ns": 503667.0, "calibration_gpu_ticks": 503667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 48291.0, "compute_span_ns": 48291.0, "command_buffer_ns": 48250.018153339624, "calibration_cpu_ns": 558292.0, "calibration_gpu_ticks": 558292.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49750.0, "compute_span_ns": 49750.0, "command_buffer_ns": 49749.971367418766, "calibration_cpu_ns": 492917.0, "calibration_gpu_ticks": 492917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 50334.0, "compute_span_ns": 50334.0, "command_buffer_ns": 50375.00523030758, "calibration_cpu_ns": 523666.0, "calibration_gpu_ticks": 523666.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 51625.0, "compute_span_ns": 51625.0, "command_buffer_ns": 51625.01474842429, "calibration_cpu_ns": 465834.0, "calibration_gpu_ticks": 465834.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 47583.0, "compute_span_ns": 47583.0, "command_buffer_ns": 47541.68912768364, "calibration_cpu_ns": 574500.0, "calibration_gpu_ticks": 574500.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 49792.0, "compute_span_ns": 49792.0, "command_buffer_ns": 49833.32473784685, "calibration_cpu_ns": 474917.0, "calibration_gpu_ticks": 474917.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 47.293046875, 46.396453125, 47.26175, 47.454375, 47.23121875, 47.144578125, 47.10346875, 47.630234375, 47.19084375 ], "compute_throughput_us_p50": 47.23121875, "compute_throughput_us_p90": 47.489546875, "command_buffer_throughput_us": [ 59.480469644768164, 57.91536477772752, 59.64648607914569, 59.14648681937251, 60.41015603841515, 59.52408901066519, 60.25976472301409, 60.51757827663096, 59.630859141179826 ], "command_buffer_throughput_us_p50": 59.630859141179826, "command_buffer_throughput_us_p90": 60.43164048605831, "compute_latency_us": [ 47.083, 49.917, 52.209, 48.291, 49.75, 50.334, 51.625, 47.583, 49.792 ], "compute_latency_us_p50": 49.792, "compute_latency_us_p90": 51.7418, "command_buffer_latency_us": [ 47.083303797990084, 49.958296585828066, 52.24999040365219, 48.250018153339624, 49.749971367418766, 50.37500523030758, 51.62501474842429, 47.54168912768364, 49.83332473784685 ], "command_buffer_latency_us_p50": 49.83332473784685, "command_buffer_latency_us_p90": 51.75000987946987, "counter_control_throughput_ratio": 2.0858486831164424, "counter_control_latency_ratio": 1.0151134653528058 }, "download_ms": 0.367, "allocation_upload_ms": 0.885875, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 2.673614807194649e-10, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "256", "3072", "1", "1", "3072", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-uta2q8a3/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7bac43b2781c01105a8e678fa6161555c0e89cbd2b0a98679fdc42bb1bd74b45", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.28925, "capture_ms": 0.064209, "compile_ms": 51.185625, "allocation_upload_ms": 1.866833, "cold_call_ms": 5.118792, "warmup_ms": 200.396542, "download_ms": 0.860125, "repetitions": 1553, "output_elements": 786432, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 15.4144932389, 15.3742491951, 15.5245492595, 14.6984063104, 14.6168705731, 14.9611500322, 14.505499678, 14.7282949131, 14.5037025113 ], "latency_us": [ 267.208, 239.708, 212, 215.459, 220.334, 222.833, 238.375, 234.458, 250.708 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 256, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 128, "reduction_scalar_rounds": 60, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 12288, "global_write_bytes": 12288, "private_read_bytes": 73728, "private_write_bytes": 24576 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 48, "global_write_bytes": 48, "private_read_bytes": 288, "private_write_bytes": 96 }, "striped_storage_scalars_per_worker": 24, "reduction_operations": 2, "reduction_elements": 6144, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 100, "normalized_kernel_cost": 400, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 9216, "fragment_scalars_per_lane": 0, "concurrent_waves": 4, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 911334, "compute_span_ns": 911334, "calibration_cpu_ns": 1393958, "calibration_gpu_ticks": 1393958, "compute_passes": 1, "command_buffer_ns": 911416.660529, "command_buffers": 2 }, { "compute_ns": 892333, "compute_span_ns": 892333, "calibration_cpu_ns": 1319666, "calibration_gpu_ticks": 1319666, "compute_passes": 1, "command_buffer_ns": 892416.690476, "command_buffers": 2 }, { "compute_ns": 933083, "compute_span_ns": 933083, "calibration_cpu_ns": 1507500, "calibration_gpu_ticks": 1507500, "compute_passes": 1, "command_buffer_ns": 933124.916628, "command_buffers": 2 }, { "compute_ns": 958750, "compute_span_ns": 958750, "calibration_cpu_ns": 1441917, "calibration_gpu_ticks": 1441917, "compute_passes": 1, "command_buffer_ns": 958874.996286, "command_buffers": 2 }, { "compute_ns": 909583, "compute_span_ns": 909583, "calibration_cpu_ns": 1392834, "calibration_gpu_ticks": 1392834, "compute_passes": 1, "command_buffer_ns": 909624.970518, "command_buffers": 2 }, { "compute_ns": 970458, "compute_span_ns": 970458, "calibration_cpu_ns": 1448917, "calibration_gpu_ticks": 1448917, "compute_passes": 1, "command_buffer_ns": 970499.997493, "command_buffers": 2 }, { "compute_ns": 908041, "compute_span_ns": 908041, "calibration_cpu_ns": 1417958, "calibration_gpu_ticks": 1417958, "compute_passes": 1, "command_buffer_ns": 908041.605726, "command_buffers": 2 }, { "compute_ns": 971959, "compute_span_ns": 971959, "calibration_cpu_ns": 1459250, "calibration_gpu_ticks": 1459250, "compute_passes": 1, "command_buffer_ns": 972083.362285, "command_buffers": 2 }, { "compute_ns": 910250, "compute_span_ns": 910250, "calibration_cpu_ns": 1333208, "calibration_gpu_ticks": 1333208, "compute_passes": 1, "command_buffer_ns": 910250.004381, "command_buffers": 2 } ], "latency": [ { "compute_ns": 18750, "compute_span_ns": 18750, "calibration_cpu_ns": 410083, "calibration_gpu_ticks": 410083, "compute_passes": 1, "command_buffer_ns": 18750.0263564, "command_buffers": 2 }, { "compute_ns": 18583, "compute_span_ns": 18583, "calibration_cpu_ns": 426875, "calibration_gpu_ticks": 426875, "compute_passes": 1, "command_buffer_ns": 18666.6147783, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 407584, "calibration_gpu_ticks": 407584, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "compute_ns": 18541, "compute_span_ns": 18541, "calibration_cpu_ns": 401417, "calibration_gpu_ticks": 401417, "compute_passes": 1, "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "compute_ns": 18584, "compute_span_ns": 18584, "calibration_cpu_ns": 451292, "calibration_gpu_ticks": 451292, "compute_passes": 1, "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "compute_ns": 18584, "compute_span_ns": 18584, "calibration_cpu_ns": 411083, "calibration_gpu_ticks": 411083, "compute_passes": 1, "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 436208, "calibration_gpu_ticks": 436208, "compute_passes": 1, "command_buffer_ns": 18499.9662451, "command_buffers": 2 }, { "compute_ns": 18500, "compute_span_ns": 18500, "calibration_cpu_ns": 401167, "calibration_gpu_ticks": 401167, "compute_passes": 1, "command_buffer_ns": 18500.0244528, "command_buffers": 2 }, { "compute_ns": 18541, "compute_span_ns": 18541, "calibration_cpu_ns": 478708, "calibration_gpu_ticks": 478708, "compute_passes": 1, "command_buffer_ns": 18583.3196156, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 978791.67879, "command_buffers": 2 }, { "command_buffer_ns": 890791.707207, "command_buffers": 2 }, { "command_buffer_ns": 961208.308581, "command_buffers": 2 }, { "command_buffer_ns": 895541.685168, "command_buffers": 2 }, { "command_buffer_ns": 903541.687876, "command_buffers": 2 }, { "command_buffer_ns": 930041.715037, "command_buffers": 2 }, { "command_buffer_ns": 920749.967918, "command_buffers": 2 }, { "command_buffer_ns": 931791.670155, "command_buffers": 2 }, { "command_buffer_ns": 903125.037439, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 18875.0564121, "command_buffers": 2 }, { "command_buffer_ns": 18541.5847227, "command_buffers": 2 }, { "command_buffer_ns": 18750.0263564, "command_buffers": 2 }, { "command_buffer_ns": 18624.9963008, "command_buffers": 2 }, { "command_buffer_ns": 18833.3215192, "command_buffers": 2 }, { "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "command_buffer_ns": 18541.6429304, "command_buffers": 2 }, { "command_buffer_ns": 18583.3196156, "command_buffers": 2 }, { "command_buffer_ns": 18666.672986, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 15.29361998109375, 13.918620425109374, 15.018879821578125, 13.992838830750001, 14.1178388730625, 14.531901797453125, 14.386718248718749, 14.559244846171875, 14.111328709984374 ], "command_buffer_throughput_us_p50": 14.386718248718749, "command_buffer_throughput_us_p90": 15.07382785348125, "command_buffer_latency_us": [ 18.875056412099998, 18.5415847227, 18.750026356400003, 18.624996300800003, 18.8333215192, 18.541642930400002, 18.541642930400002, 18.5833196156, 18.666672986000002 ], "command_buffer_latency_us_p50": 18.624996300800003, "command_buffer_latency_us_p90": 18.84166849778 }, "compute_throughput_us": [ 14.23959375, 13.942703125, 14.579421875, 14.98046875, 14.212234375, 15.16340625, 14.188140625, 15.186859375, 14.22265625 ], "compute_throughput_us_p50": 14.23959375, "compute_throughput_us_p90": 15.168096875, "command_buffer_throughput_us": [ 14.240885320765624, 13.9440107886875, 14.580076822312499, 14.98242181696875, 14.212890164343749, 15.164062460828125, 14.188150089468751, 15.188802535703125, 14.222656318453124 ], "command_buffer_throughput_us_p50": 14.240885320765624, "command_buffer_throughput_us_p90": 15.169010475803125, "compute_latency_us": [ 18.75, 18.583, 18.5, 18.541, 18.584, 18.584, 18.5, 18.5, 18.541 ], "compute_latency_us_p50": 18.541, "compute_latency_us_p90": 18.6172, "command_buffer_latency_us": [ 18.750026356400003, 18.6666147783, 18.541642930400002, 18.541642930400002, 18.624996300800003, 18.624996300800003, 18.4999662451, 18.5000244528, 18.5833196156 ], "command_buffer_latency_us_p50": 18.5833196156, "command_buffer_latency_us_p90": 18.68329709392, "counter_control_throughput_ratio": 1.0067327083228448, "counter_control_latency_ratio": 0.9955246503648207 }, "correctness": { "max_abs_error": 1.569250362249905e-10, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 14.7282949131, "throughput_us_p90": 15.43650444302, "latency_us_p50": 234.458, "latency_us_p90": 254.008 }, "slowdown": 0.4376297217640049, "device_compute_slowdown": 0.301486900547109, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "native_first", "implementation_order": [ "native", "torch" ], "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-v4w_vpk0/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "512", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "2", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "5e168786762f89acc561194a669a3725da4fe77bed5800b1d573576d096bb3fc", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 512, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 17.817417, "capture_ms": 0.069833, "compile_ms": 48.831583, "allocation_upload_ms": 5.784166, "cold_call_ms": 6.662791, "warmup_ms": 200.51025, "download_ms": 3.605375, "repetitions": 454, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 63.3773854626, 63.5185396476, 63.513123348, 62.9382356828, 63.293685022, 63.3784867841, 63.0260660793, 63.127660793, 62.9814603524 ], "latency_us": [ 349.125, 308.708, 272.5, 278.916, 280.291, 270.458, 289.791, 279.5, 261.792 ], "execution_plans": [ { "threads": 512, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 128, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 2, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 384, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 160, "normalized_kernel_cost": 1920, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 4031709, "compute_span_ns": 4031709, "calibration_cpu_ns": 4542167, "calibration_gpu_ticks": 4542167, "compute_passes": 1, "command_buffer_ns": 4031874.99614, "command_buffers": 2 }, { "compute_ns": 3993167, "compute_span_ns": 3993167, "calibration_cpu_ns": 4444541, "calibration_gpu_ticks": 4444541, "compute_passes": 1, "command_buffer_ns": 3993250.02218, "command_buffers": 2 }, { "compute_ns": 3993708, "compute_span_ns": 3993708, "calibration_cpu_ns": 4469000, "calibration_gpu_ticks": 4469000, "compute_passes": 1, "command_buffer_ns": 3993708.29109, "command_buffers": 2 }, { "compute_ns": 3994375, "compute_span_ns": 3994375, "calibration_cpu_ns": 4454250, "calibration_gpu_ticks": 4454250, "compute_passes": 1, "command_buffer_ns": 3994416.62012, "command_buffers": 2 }, { "compute_ns": 3974333, "compute_span_ns": 3974333, "calibration_cpu_ns": 4417500, "calibration_gpu_ticks": 4417500, "compute_passes": 1, "command_buffer_ns": 3974333.34729, "command_buffers": 2 }, { "compute_ns": 4087209, "compute_span_ns": 4087209, "calibration_cpu_ns": 4542209, "calibration_gpu_ticks": 4542209, "compute_passes": 1, "command_buffer_ns": 4087249.98124, "command_buffers": 2 }, { "compute_ns": 4094000, "compute_span_ns": 4094000, "calibration_cpu_ns": 4553333, "calibration_gpu_ticks": 4553333, "compute_passes": 1, "command_buffer_ns": 4093999.97443, "command_buffers": 2 }, { "compute_ns": 4010459, "compute_span_ns": 4010459, "calibration_cpu_ns": 4438750, "calibration_gpu_ticks": 4438750, "compute_passes": 1, "command_buffer_ns": 4010541.7138, "command_buffers": 2 }, { "compute_ns": 3971917, "compute_span_ns": 3971917, "calibration_cpu_ns": 4448375, "calibration_gpu_ticks": 4448375, "compute_passes": 1, "command_buffer_ns": 3972000.03499, "command_buffers": 2 } ], "latency": [ { "compute_ns": 63291, "compute_span_ns": 63291, "calibration_cpu_ns": 468125, "calibration_gpu_ticks": 468125, "compute_passes": 1, "command_buffer_ns": 63333.3693258, "command_buffers": 2 }, { "compute_ns": 63333, "compute_span_ns": 63333, "calibration_cpu_ns": 435833, "calibration_gpu_ticks": 435833, "compute_passes": 1, "command_buffer_ns": 63375.0460111, "command_buffers": 2 }, { "compute_ns": 63292, "compute_span_ns": 63292, "calibration_cpu_ns": 487375, "calibration_gpu_ticks": 487375, "compute_passes": 1, "command_buffer_ns": 63374.9878034, "command_buffers": 2 }, { "compute_ns": 63167, "compute_span_ns": 63167, "calibration_cpu_ns": 537875, "calibration_gpu_ticks": 537875, "compute_passes": 1, "command_buffer_ns": 63249.9577478, "command_buffers": 2 }, { "compute_ns": 63500, "compute_span_ns": 63500, "calibration_cpu_ns": 441833, "calibration_gpu_ticks": 441833, "compute_passes": 1, "command_buffer_ns": 63541.5781289, "command_buffers": 2 }, { "compute_ns": 62875, "compute_span_ns": 62875, "calibration_cpu_ns": 449917, "calibration_gpu_ticks": 449917, "compute_passes": 1, "command_buffer_ns": 62916.718889, "command_buffers": 2 }, { "compute_ns": 63125, "compute_span_ns": 63125, "calibration_cpu_ns": 499250, "calibration_gpu_ticks": 499250, "compute_passes": 1, "command_buffer_ns": 63208.3392702, "command_buffers": 2 }, { "compute_ns": 63458, "compute_span_ns": 63458, "calibration_cpu_ns": 451792, "calibration_gpu_ticks": 451792, "compute_passes": 1, "command_buffer_ns": 63500.017859, "command_buffers": 2 }, { "compute_ns": 63583, "compute_span_ns": 63583, "calibration_cpu_ns": 428125, "calibration_gpu_ticks": 428125, "compute_passes": 1, "command_buffer_ns": 63583.3130218, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 4082166.648, "command_buffers": 2 }, { "command_buffer_ns": 4026875.01628, "command_buffers": 2 }, { "command_buffer_ns": 4064249.98073, "command_buffers": 2 }, { "command_buffer_ns": 4097583.35445, "command_buffers": 2 }, { "command_buffer_ns": 4011791.66511, "command_buffers": 2 }, { "command_buffer_ns": 4007875.04623, "command_buffers": 2 }, { "command_buffer_ns": 3986041.64366, "command_buffers": 2 }, { "command_buffer_ns": 4015499.95877, "command_buffers": 2 }, { "command_buffer_ns": 4090791.68458, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 63874.9916106, "command_buffers": 2 }, { "command_buffer_ns": 63416.6644886, "command_buffers": 2 }, { "command_buffer_ns": 63458.3411738, "command_buffers": 2 }, { "command_buffer_ns": 82333.3975859, "command_buffers": 2 }, { "command_buffer_ns": 77083.3576098, "command_buffers": 2 }, { "command_buffer_ns": 66833.2795613, "command_buffers": 2 }, { "command_buffer_ns": 63416.6644886, "command_buffers": 2 }, { "command_buffer_ns": 77583.3614171, "command_buffers": 2 }, { "command_buffer_ns": 76833.3557062, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 63.783853875, 62.919922129375, 63.50390594890625, 64.02473991328125, 62.68424476734375, 62.62304759734375, 62.2819006821875, 62.74218685578125, 63.9186200715625 ], "command_buffer_throughput_us_p50": 62.919922129375, "command_buffer_throughput_us_p90": 63.93984403990625, "command_buffer_latency_us": [ 63.8749916106, 63.4166644886, 63.4583411738, 82.3333975859, 77.0833576098, 66.8332795613, 63.4166644886, 77.5833614171, 76.83335570620001 ], "command_buffer_latency_us_p50": 66.8332795613, "command_buffer_latency_us_p90": 78.53336865086 }, "compute_throughput_us": [ 62.995453125, 62.393234375, 62.4016875, 62.412109375, 62.098953125, 63.862640625, 63.96875, 62.663421875, 62.061203125 ], "compute_throughput_us_p50": 62.412109375, "compute_throughput_us_p90": 63.8838625, "command_buffer_throughput_us": [ 62.9980468146875, 62.394531596562494, 62.40169204828125, 62.412759689375, 62.09895855140625, 63.863280956875, 63.96874960046875, 62.664714278125004, 62.06250054671875 ], "command_buffer_throughput_us_p50": 62.412759689375, "command_buffer_throughput_us_p90": 63.88437468559375, "compute_latency_us": [ 63.291, 63.333, 63.292, 63.167, 63.5, 62.875, 63.125, 63.458, 63.583 ], "compute_latency_us_p50": 63.292, "compute_latency_us_p90": 63.5166, "command_buffer_latency_us": [ 63.3333693258, 63.3750460111, 63.374987803399996, 63.249957747799996, 63.5415781289, 62.916718889, 63.2083392702, 63.500017858999996, 63.5833130218 ], "command_buffer_latency_us_p50": 63.374987803399996, "command_buffer_latency_us_p90": 63.54992510748, "counter_control_throughput_ratio": 0.9906629454002386, "counter_control_latency_ratio": 0.9413980475294842 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 63.293685022, "throughput_us_p90": 63.51420660792, "latency_us_p50": 279.5, "latency_us_p90": 316.7914 }, "torch": { "cold_call_ms": 7.0635, "warmup_ms": 201.478709, "repetitions": 188, "throughput_us": [ 131.59308510638297, 131.78745744680853, 132.57358510638298, 132.04232978723405, 131.84995212765955, 132.0474308510638, 132.47340425531914, 132.71476063829786, 131.1852819148936 ], "latency_us": [ 630.6659999999999, 440.458, 426.70799999999997, 410.95799999999997, 409.125, 425.25, 424.375, 420.459, 546.417 ], "throughput_us_p50": 132.04232978723405, "throughput_us_p90": 132.60182021276594, "latency_us_p50": 425.25, "latency_us_p90": 563.2668, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7877125.055529177, "command_buffers": 11 }, { "command_buffer_ns": 7911916.64757207, "command_buffers": 11 }, { "command_buffer_ns": 7885916.6242182255, "command_buffers": 11 }, { "command_buffer_ns": 7870875.007938594, "command_buffers": 11 }, { "command_buffer_ns": 7804416.643921286, "command_buffers": 11 }, { "command_buffer_ns": 7922750.141005963, "command_buffers": 11 }, { "command_buffer_ns": 7887999.99281764, "command_buffers": 11 }, { "command_buffer_ns": 7910583.3428911865, "command_buffers": 11 }, { "command_buffer_ns": 7881833.298597485, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 134166.67934507132, "command_buffers": 1 }, { "command_buffer_ns": 152666.64559021592, "command_buffers": 1 }, { "command_buffer_ns": 137375.02740696073, "command_buffers": 1 }, { "command_buffer_ns": 135749.98592957854, "command_buffers": 1 }, { "command_buffer_ns": 141250.02780929208, "command_buffers": 1 }, { "command_buffer_ns": 150875.0137872994, "command_buffers": 1 }, { "command_buffer_ns": 134250.0327154994, "command_buffers": 1 }, { "command_buffer_ns": 137125.02550333738, "command_buffers": 1 }, { "command_buffer_ns": 158499.98453631997, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 123.0800789926434, 123.6236976183136, 123.21744725340977, 122.98242199904053, 121.94401006127009, 123.79297095321817, 123.24999988777563, 123.60286473267479, 123.1536452905857 ], "command_buffer_throughput_us_p50": 123.21744725340977, "command_buffer_throughput_us_p90": 123.65755228529451, "command_buffer_latency_us": [ 134.16667934507132, 152.66664559021592, 137.37502740696073, 135.74998592957854, 141.25002780929208, 150.8750137872994, 134.2500327154994, 137.12502550333738, 158.49998453631997 ], "command_buffer_latency_us_p50": 137.37502740696073, "command_buffer_latency_us_p90": 153.83331337943673 }, "throughput": [ { "compute_ns": 8785209.0, "compute_span_ns": 10422041.0, "command_buffer_ns": 10154625.051654875, "calibration_cpu_ns": 11426792.0, "calibration_gpu_ticks": 11426792.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8719958.0, "compute_span_ns": 10393500.0, "command_buffer_ns": 10132124.996744096, "calibration_cpu_ns": 11248291.0, "calibration_gpu_ticks": 11248291.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8658247.0, "compute_span_ns": 10281166.0, "command_buffer_ns": 10000458.278227597, "calibration_cpu_ns": 10964750.0, "calibration_gpu_ticks": 10964750.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8653376.0, "compute_span_ns": 10267584.0, "command_buffer_ns": 10030333.301983774, "calibration_cpu_ns": 10994833.0, "calibration_gpu_ticks": 10994833.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8694837.0, "compute_span_ns": 10307709.0, "command_buffer_ns": 10052166.471723467, "calibration_cpu_ns": 11249417.0, "calibration_gpu_ticks": 11249417.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8609673.0, "compute_span_ns": 10212500.0, "command_buffer_ns": 9953458.386007696, "calibration_cpu_ns": 11059208.0, "calibration_gpu_ticks": 11059208.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8656463.0, "compute_span_ns": 10291125.0, "command_buffer_ns": 10016083.426307887, "calibration_cpu_ns": 11367291.0, "calibration_gpu_ticks": 11367291.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8689796.0, "compute_span_ns": 10386375.0, "command_buffer_ns": 10143166.524358094, "calibration_cpu_ns": 11209250.0, "calibration_gpu_ticks": 11209250.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8673872.0, "compute_span_ns": 10325959.0, "command_buffer_ns": 10060166.707262397, "calibration_cpu_ns": 11013375.0, "calibration_gpu_ticks": 11013375.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 133625.0, "compute_span_ns": 133625.0, "command_buffer_ns": 133624.9988526106, "calibration_cpu_ns": 671375.0, "calibration_gpu_ticks": 671375.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135291.0, "compute_span_ns": 135291.0, "command_buffer_ns": 135250.04032999277, "calibration_cpu_ns": 593000.0, "calibration_gpu_ticks": 593000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 134250.0, "compute_span_ns": 134250.0, "command_buffer_ns": 134249.9745078385, "calibration_cpu_ns": 639917.0, "calibration_gpu_ticks": 639917.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 143083.0, "compute_span_ns": 143083.0, "command_buffer_ns": 143083.33629742265, "calibration_cpu_ns": 624042.0, "calibration_gpu_ticks": 624042.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136417.0, "compute_span_ns": 136417.0, "command_buffer_ns": 136416.6964776814, "calibration_cpu_ns": 618125.0, "calibration_gpu_ticks": 618125.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 150208.0, "compute_span_ns": 150208.0, "command_buffer_ns": 150166.6847616434, "calibration_cpu_ns": 630083.0, "calibration_gpu_ticks": 630083.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 138084.0, "compute_span_ns": 138084.0, "command_buffer_ns": 138124.97491016984, "calibration_cpu_ns": 607000.0, "calibration_gpu_ticks": 607000.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 143334.0, "compute_span_ns": 143334.0, "command_buffer_ns": 143375.01488626003, "calibration_cpu_ns": 603667.0, "calibration_gpu_ticks": 603667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 133750.0, "compute_span_ns": 133750.0, "command_buffer_ns": 133749.9707005918, "calibration_cpu_ns": 609000.0, "calibration_gpu_ticks": 609000.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 137.268890625, 136.24934375, 135.285109375, 135.209, 135.856828125, 134.526140625, 135.257234375, 135.7780625, 135.52925 ], "compute_throughput_us_p50": 135.52925, "compute_throughput_us_p90": 136.453253125, "command_buffer_throughput_us": [ 158.66601643210743, 158.3144530741265, 156.2571605973062, 156.72395784349646, 157.06510112067917, 155.52278728137026, 156.50130353606073, 158.48697694309521, 157.19010480097495 ], "command_buffer_throughput_us_p50": 157.06510112067917, "command_buffer_throughput_us_p90": 158.52278484089766, "compute_latency_us": [ 133.625, 135.291, 134.25, 143.083, 136.417, 150.208, 138.084, 143.334, 133.75 ], "compute_latency_us_p50": 136.417, "compute_latency_us_p90": 144.7088, "command_buffer_latency_us": [ 133.6249988526106, 135.25004032999277, 134.2499745078385, 143.08333629742265, 136.4166964776814, 150.1666847616434, 138.12497491016984, 143.37501488626003, 133.7499707005918 ], "command_buffer_latency_us_p50": 136.4166964776814, "command_buffer_latency_us_p90": 144.7333488613367, "counter_control_throughput_ratio": 1.2763739508487868, "counter_control_latency_ratio": 0.9953051932995707 }, "download_ms": 0.62675, "allocation_upload_ms": 16.66, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "slowdown": 0.47934389770301733, "device_compute_slowdown": 0.46050656500349557, "valid": true, "round": 3, "variant": "candidate" }, { "case": { "operation": "softmax", "m": 768, "n": 6144, "k": 1 }, "name": "softmax_768x6144", "backend": "metal", "block": [ 1, 6144, 1 ], "timing_order": "torch_first", "implementation_order": [ "torch", "native" ], "torch": { "cold_call_ms": 0.637208, "warmup_ms": 201.140709, "repetitions": 225, "throughput_us": [ 130.52944444444444, 129.88870222222224, 130.23036888888888, 129.77777777777777, 131.4407422222222, 129.17388888888888, 130.185, 130.87129777777776, 130.67759555555554 ], "latency_us": [ 436.541, 563.7919999999999, 402.45799999999997, 388.291, 380.33299999999997, 369.958, 401.875, 449.5, 385.542 ], "throughput_us_p50": 130.23036888888888, "throughput_us_p90": 130.98518666666664, "latency_us_p50": 401.875, "latency_us_p90": 472.3584, "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 7796083.40235427, "command_buffers": 11 }, { "command_buffer_ns": 7875874.871388078, "command_buffers": 11 }, { "command_buffer_ns": 7862958.300393075, "command_buffers": 11 }, { "command_buffer_ns": 7830041.751731187, "command_buffers": 11 }, { "command_buffer_ns": 7825499.982573092, "command_buffers": 11 }, { "command_buffer_ns": 7861041.638534516, "command_buffers": 11 }, { "command_buffer_ns": 7893333.4443718195, "command_buffers": 11 }, { "command_buffer_ns": 7860208.512283862, "command_buffers": 11 }, { "command_buffer_ns": 7837833.371013403, "command_buffers": 11 } ], "latency": [ { "command_buffer_ns": 138000.00306218863, "command_buffers": 1 }, { "command_buffer_ns": 136666.64017364383, "command_buffers": 1 }, { "command_buffer_ns": 134375.00456348062, "command_buffers": 1 }, { "command_buffer_ns": 147124.98523294926, "command_buffers": 1 }, { "command_buffer_ns": 134125.00265985727, "command_buffers": 1 }, { "command_buffer_ns": 136999.99544769526, "command_buffers": 1 }, { "command_buffer_ns": 151749.99134615064, "command_buffers": 1 }, { "command_buffer_ns": 136291.66642203927, "command_buffers": 1 }, { "command_buffer_ns": 133624.9988526106, "command_buffers": 1 } ], "command_buffer_throughput_us": [ 121.81380316178547, 123.06054486543871, 122.8587234436418, 122.3444023707998, 122.27343722770456, 122.82877560210181, 123.33333506830968, 122.81575800443534, 122.46614642208442 ], "command_buffer_throughput_us_p50": 122.81575800443534, "command_buffer_throughput_us_p90": 123.11510290601291, "command_buffer_latency_us": [ 138.00000306218863, 136.66664017364383, 134.37500456348062, 147.12498523294926, 134.12500265985727, 136.99999544769526, 151.74999134615064, 136.29166642203927, 133.6249988526106 ], "command_buffer_latency_us_p50": 136.66664017364383, "command_buffer_latency_us_p90": 148.04998645558953 }, "throughput": [ { "compute_ns": 8630670.0, "compute_span_ns": 10273584.0, "command_buffer_ns": 10019666.74812138, "calibration_cpu_ns": 11067875.0, "calibration_gpu_ticks": 11067875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8664540.0, "compute_span_ns": 10344667.0, "command_buffer_ns": 10055041.639134288, "calibration_cpu_ns": 11009125.0, "calibration_gpu_ticks": 11009125.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8581745.0, "compute_span_ns": 10201833.0, "command_buffer_ns": 9948333.259671926, "calibration_cpu_ns": 11050958.0, "calibration_gpu_ticks": 11050958.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8653543.0, "compute_span_ns": 10209250.0, "command_buffer_ns": 9957500.034943223, "calibration_cpu_ns": 10876958.0, "calibration_gpu_ticks": 10876958.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8629210.0, "compute_span_ns": 10184917.0, "command_buffer_ns": 9920416.574459523, "calibration_cpu_ns": 10935875.0, "calibration_gpu_ticks": 10935875.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8692504.0, "compute_span_ns": 10372334.0, "command_buffer_ns": 10095416.859257966, "calibration_cpu_ns": 11063791.0, "calibration_gpu_ticks": 11063791.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8659083.0, "compute_span_ns": 10250458.0, "command_buffer_ns": 10023791.692219675, "calibration_cpu_ns": 11131083.0, "calibration_gpu_ticks": 11131083.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8702043.0, "compute_span_ns": 10319958.0, "command_buffer_ns": 10047833.260614425, "calibration_cpu_ns": 11102167.0, "calibration_gpu_ticks": 11102167.0, "compute_passes": 64, "command_buffers": 11 }, { "compute_ns": 8616577.0, "compute_span_ns": 10210916.0, "command_buffer_ns": 9956333.320587873, "calibration_cpu_ns": 10996958.0, "calibration_gpu_ticks": 10996958.0, "compute_passes": 64, "command_buffers": 11 } ], "latency": [ { "compute_ns": 137125.0, "compute_span_ns": 137125.0, "command_buffer_ns": 137124.96729567647, "calibration_cpu_ns": 642625.0, "calibration_gpu_ticks": 642625.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 141125.0, "compute_span_ns": 141125.0, "command_buffer_ns": 141124.99775364995, "calibration_cpu_ns": 643667.0, "calibration_gpu_ticks": 643667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 137750.0, "compute_span_ns": 137750.0, "command_buffer_ns": 137750.00115856528, "calibration_cpu_ns": 714250.0, "calibration_gpu_ticks": 714250.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 154042.0, "compute_span_ns": 154042.0, "command_buffer_ns": 154041.68516397476, "calibration_cpu_ns": 627583.0, "calibration_gpu_ticks": 627583.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 142333.0, "compute_span_ns": 142333.0, "command_buffer_ns": 142333.33058655262, "calibration_cpu_ns": 900667.0, "calibration_gpu_ticks": 900667.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 134792.0, "compute_span_ns": 134792.0, "command_buffer_ns": 134791.71320796013, "calibration_cpu_ns": 550167.0, "calibration_gpu_ticks": 550167.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 145833.0, "compute_span_ns": 145833.0, "command_buffer_ns": 145791.62234440446, "calibration_cpu_ns": 620417.0, "calibration_gpu_ticks": 620417.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 135375.0, "compute_span_ns": 135375.0, "command_buffer_ns": 135375.012177974, "calibration_cpu_ns": 743750.0, "calibration_gpu_ticks": 743750.0, "compute_passes": 1, "command_buffers": 1 }, { "compute_ns": 136917.0, "compute_span_ns": 136917.0, "command_buffer_ns": 136916.64207726717, "calibration_cpu_ns": 687208.0, "calibration_gpu_ticks": 687208.0, "compute_passes": 1, "command_buffers": 1 } ], "compute_throughput_us": [ 134.85421875, 135.3834375, 134.089765625, 135.211609375, 134.83140625, 135.820375, 135.298171875, 135.969421875, 134.634015625 ], "compute_throughput_us_p50": 135.211609375, "compute_throughput_us_p90": 135.850184375, "command_buffer_throughput_us": [ 156.55729293939658, 157.11002561147325, 155.44270718237385, 155.58593804598786, 155.00650897593005, 157.74088842590572, 156.62174519093242, 156.9973946971004, 155.56770813418552 ], "command_buffer_throughput_us_p50": 156.55729293939658, "command_buffer_throughput_us_p90": 157.23619817435974, "compute_latency_us": [ 137.125, 141.125, 137.75, 154.042, 142.333, 134.792, 145.833, 135.375, 136.917 ], "compute_latency_us_p50": 137.75, "compute_latency_us_p90": 147.4748, "command_buffer_latency_us": [ 137.12496729567647, 141.12499775364995, 137.75000115856528, 154.04168516397476, 142.33333058655262, 134.79171320796013, 145.79162234440446, 135.37501217797399, 136.91664207726717 ], "command_buffer_latency_us_p50": 137.75000115856528, "command_buffer_latency_us_p90": 147.44163490831852, "counter_control_throughput_ratio": 1.2717045899201833, "counter_control_latency_ratio": 1.0246334387496405 }, "download_ms": 0.632666, "allocation_upload_ms": 0.953791, "device": "mps:0", "output_policy": "preallocated_out", "correctness": { "max_abs_error": 7.553236784944503e-11, "atol": 2e-06, "rtol": 2e-05 } }, "native_command": [ "/Users/mike/CLionProjects/luisa/cmake-build-tirx/bin/benchmark_tile_tirx", "metal", "softmax", "768", "6144", "1", "1", "6144", "1", "9", "30", "200", "/var/folders/2_/k4txnts916qb_hd5bnch_jfw0000gn/T/luisa-tile-benchmark-iwukpe9t/output.f32", "auto", "2", "subgroup-reduce", "vectorize", "256", "1", "tvm", "retain-subgroup-fences", "0", "16", "retain-input-snapshots", "generic", "reference", "reference", "preserve", "1", "auto", "1", "4", "cache" ], "native_stderr": "", "native_source_sha256": "7f45b1ebfa78e24e3d477c2ad514fde5c49a3d7b63ed786757d9dc5d1979c8b1", "native": { "backend": "metal", "operation": "softmax", "execution_scope": "auto", "pipeline_window": 2, "cooperative_matrix": false, "metal_mpp": false, "metal_subgroup_reductions": true, "metal_max_threads": 1024, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "cache_reduction_inputs": true, "reduction_cost_profile": "analytic", "fuse_gpu_elementwise": true, "shared_tile_materialization": "preserve", "forward_readonly_tile_loads": true, "elide_independent_subgroup_barriers": false, "vectorize": true, "auto_vectorize": false, "cpu_stack_bytes": 0, "cpu_parallel_task_threshold": 64, "cpu_vector_lanes": 16, "max_reduction_striped_scalars_per_worker": 64, "cpu_target_policy": "generic", "cpu_model": "generic", "cpu_matrix_backend": "reference", "cpu_math_backend": "reference", "planner_threads": 256, "copy_batch": 1, "matrix_intrinsics": 0, "external_matrix_calls": 0, "external_vector_math_calls": 0, "simdgroup_intrinsics": 0, "mpp_intrinsics": 0, "runtime_init_ms": 20.595042, "capture_ms": 0.061916, "compile_ms": 48.080709, "allocation_upload_ms": 6.005708, "cold_call_ms": 4.810209, "warmup_ms": 200.030833, "download_ms": 3.210458, "repetitions": 446, "output_elements": 4718592, "mma_operations": 0, "compiler": "21.0.0 (clang-2100.1.1.101)", "throughput_us": [ 57.7531748879, 57.7215067265, 56.7023542601, 57.9460941704, 56.7273901345, 56.6120156951, 56.71132287, 57.5671704036, 56.5948251121 ], "latency_us": [ 353.667, 287.583, 277.584, 285.667, 323.625, 276.208, 337.458, 374.125, 284.625 ], "execution_plans": [ { "threads": 256, "metal_mpp": false, "cost_basis": "metal_subgroup_reduction_v1", "programs": 768, "shared_memory_bytes": 64, "optimized": true, "candidates_considered": 1, "candidates_rejected": 0, "max_copy_batch": 1, "batched_copy_operations": 0, "prefetched_pipeline_loops": 0, "prefetch_storage_scalars_per_lane": 0, "reduction_subgroups_per_program": 8, "reduction_programs_per_group": 1, "reduction_unroll_factor": 1, "reduction_lane_elements": 4, "reduction_threadgroups": 768, "reduction_scalar_rounds": 120, "reduction_lane_utilization": 1, "reduction_payload_accesses_known": true, "reduction_payload_accesses_per_program": { "global_read_bytes": 24576, "global_write_bytes": 24576, "private_read_bytes": 147456, "private_write_bytes": 49152 }, "reduction_payload_accesses_per_worker": { "global_read_bytes": 96, "global_write_bytes": 96, "private_read_bytes": 576, "private_write_bytes": 192 }, "striped_storage_scalars_per_worker": 48, "reduction_operations": 2, "reduction_elements": 12288, "elementwise_elements_per_program": 0, "elementwise_scalar_temporaries": 0, "group_barrier_sites_before": 2, "group_barrier_sites_after": 2, "independent_subgroups": false, "normalized_cost": 168, "normalized_kernel_cost": 2016, "matrix_issues": 0, "shared_fragment_transfers": 0, "direct_fragment_stores": 0, "metal_mpp_operations": 0, "memory_fragment_reads": 0, "lhs_footprint_fragments": 0, "rhs_footprint_fragments": 0, "accumulator_initializations": 0, "tile_aspect_fragments": 0, "local_row_aspect_issues": 0, "local_column_aspect_issues": 0, "independent_elements": 18432, "fragment_scalars_per_lane": 0, "concurrent_waves": 12, "matrices": [] } ], "device_timing": { "method": "metal_compute_pass_timestamps_v1", "scope": "sum_of_compute_encoder_gpu_intervals", "host_samples_instrumented": false, "repetitions": 64, "throughput": [ { "compute_ns": 3742208, "compute_span_ns": 3742208, "calibration_cpu_ns": 4243709, "calibration_gpu_ticks": 4243709, "compute_passes": 1, "command_buffer_ns": 3742250.03179, "command_buffers": 2 }, { "compute_ns": 3684875, "compute_span_ns": 3684875, "calibration_cpu_ns": 4226416, "calibration_gpu_ticks": 4226416, "compute_passes": 1, "command_buffer_ns": 3684875.03147, "command_buffers": 2 }, { "compute_ns": 3664917, "compute_span_ns": 3664917, "calibration_cpu_ns": 4120583, "calibration_gpu_ticks": 4120583, "compute_passes": 1, "command_buffer_ns": 3664958.29076, "command_buffers": 2 }, { "compute_ns": 3595084, "compute_span_ns": 3595084, "calibration_cpu_ns": 4104791, "calibration_gpu_ticks": 4104791, "compute_passes": 1, "command_buffer_ns": 3595124.98835, "command_buffers": 2 }, { "compute_ns": 3543000, "compute_span_ns": 3543000, "calibration_cpu_ns": 4065125, "calibration_gpu_ticks": 4065125, "compute_passes": 1, "command_buffer_ns": 3542999.9698, "command_buffers": 2 }, { "compute_ns": 3717666, "compute_span_ns": 3717666, "calibration_cpu_ns": 4261209, "calibration_gpu_ticks": 4261209, "compute_passes": 1, "command_buffer_ns": 3717708.28497, "command_buffers": 2 }, { "compute_ns": 3563084, "compute_span_ns": 3563084, "calibration_cpu_ns": 4099750, "calibration_gpu_ticks": 4099750, "compute_passes": 1, "command_buffer_ns": 3563208.33089, "command_buffers": 2 }, { "compute_ns": 3567125, "compute_span_ns": 3567125, "calibration_cpu_ns": 4052167, "calibration_gpu_ticks": 4052167, "compute_passes": 1, "command_buffer_ns": 3567208.30314, "command_buffers": 2 }, { "compute_ns": 3577125, "compute_span_ns": 3577125, "calibration_cpu_ns": 4072500, "calibration_gpu_ticks": 4072500, "compute_passes": 1, "command_buffer_ns": 3577124.96771, "command_buffers": 2 } ], "latency": [ { "compute_ns": 87500, "compute_span_ns": 87500, "calibration_cpu_ns": 530959, "calibration_gpu_ticks": 530959, "compute_passes": 1, "command_buffer_ns": 87541.6444615, "command_buffers": 2 }, { "compute_ns": 71583, "compute_span_ns": 71583, "calibration_cpu_ns": 487250, "calibration_gpu_ticks": 487250, "compute_passes": 1, "command_buffer_ns": 71624.9924153, "command_buffers": 2 }, { "compute_ns": 70458, "compute_span_ns": 70458, "calibration_cpu_ns": 507792, "calibration_gpu_ticks": 507792, "compute_passes": 1, "command_buffer_ns": 70458.3362676, "command_buffers": 2 }, { "compute_ns": 59250, "compute_span_ns": 59250, "calibration_cpu_ns": 576625, "calibration_gpu_ticks": 576625, "compute_passes": 1, "command_buffer_ns": 59333.3388679, "command_buffers": 2 }, { "compute_ns": 72625, "compute_span_ns": 72625, "calibration_cpu_ns": 488125, "calibration_gpu_ticks": 488125, "compute_passes": 1, "command_buffer_ns": 72625.0000298, "command_buffers": 2 }, { "compute_ns": 58209, "compute_span_ns": 58209, "calibration_cpu_ns": 465041, "calibration_gpu_ticks": 465041, "compute_passes": 1, "command_buffer_ns": 58291.6545682, "command_buffers": 2 }, { "compute_ns": 58417, "compute_span_ns": 58417, "calibration_cpu_ns": 488125, "calibration_gpu_ticks": 488125, "compute_passes": 1, "command_buffer_ns": 58416.6846238, "command_buffers": 2 }, { "compute_ns": 58417, "compute_span_ns": 58417, "calibration_cpu_ns": 524875, "calibration_gpu_ticks": 524875, "compute_passes": 1, "command_buffer_ns": 58541.7146794, "command_buffers": 2 }, { "compute_ns": 71208, "compute_span_ns": 71208, "calibration_cpu_ns": 488375, "calibration_gpu_ticks": 488375, "compute_passes": 1, "command_buffer_ns": 71166.6652933, "command_buffers": 2 } ], "control": { "method": "metal_command_buffer_timestamps_v1", "scope": "sum_of_command_buffer_gpu_intervals", "encoder_instrumentation": false, "repetitions": 64, "throughput": [ { "command_buffer_ns": 3753458.32435, "command_buffers": 2 }, { "command_buffer_ns": 3542124.99224, "command_buffers": 2 }, { "command_buffer_ns": 3672958.35167, "command_buffers": 2 }, { "command_buffer_ns": 3623208.3221, "command_buffers": 2 }, { "command_buffer_ns": 3605791.65863, "command_buffers": 2 }, { "command_buffer_ns": 3555666.6553, "command_buffers": 2 }, { "command_buffer_ns": 3602833.31247, "command_buffers": 2 }, { "command_buffer_ns": 3636708.36668, "command_buffers": 2 }, { "command_buffer_ns": 3542208.34561, "command_buffers": 2 } ], "latency": [ { "command_buffer_ns": 60333.2882747, "command_buffers": 2 }, { "command_buffer_ns": 58666.6865274, "command_buffers": 2 }, { "command_buffer_ns": 69125.0315867, "command_buffers": 2 }, { "command_buffer_ns": 57541.6488573, "command_buffers": 2 }, { "command_buffer_ns": 71374.9905117, "command_buffers": 2 }, { "command_buffer_ns": 58458.3031014, "command_buffers": 2 }, { "command_buffer_ns": 74124.9532439, "command_buffers": 2 }, { "command_buffer_ns": 58833.276853, "command_buffers": 2 }, { "command_buffer_ns": 70750.0148565, "command_buffers": 2 } ], "command_buffer_throughput_us": [ 58.64778631796875, 55.34570300375, 57.38997424484375, 56.6126300328125, 56.34049466609375, 55.5572914890625, 56.29427050734375, 56.823568229375, 55.347005400156256 ], "command_buffer_throughput_us_p50": 56.34049466609375, "command_buffer_throughput_us_p90": 57.64153665946875, "command_buffer_latency_us": [ 60.333288274699996, 58.6666865274, 69.1250315867, 57.541648857300004, 71.3749905117, 58.4583031014, 74.12495324390001, 58.833276853, 70.7500148565 ], "command_buffer_latency_us_p50": 60.333288274699996, "command_buffer_latency_us_p90": 71.92498305814 }, "compute_throughput_us": [ 58.472, 57.576171875, 57.264328125, 56.1731875, 55.359375, 58.08853125, 55.6731875, 55.736328125, 55.892578125 ], "compute_throughput_us_p50": 56.1731875, "compute_throughput_us_p90": 58.165225, "command_buffer_throughput_us": [ 58.47265674671875, 57.576172366718744, 57.264973293124996, 56.17382794296876, 55.359374528125, 58.08919195265625, 55.67513017015625, 55.737629736562496, 55.89257762046875 ], "command_buffer_throughput_us_p50": 56.17382794296876, "command_buffer_throughput_us_p90": 58.16588491146875, "compute_latency_us": [ 87.5, 71.583, 70.458, 59.25, 72.625, 58.209, 58.417, 58.417, 71.208 ], "compute_latency_us_p50": 70.458, "compute_latency_us_p90": 75.60000000000001, "command_buffer_latency_us": [ 87.5416444615, 71.6249924153, 70.45833626759999, 59.3333388679, 72.62500002979999, 58.2916545682, 58.4166846238, 58.5417146794, 71.1666652933 ], "command_buffer_latency_us_p50": 70.45833626759999, "command_buffer_latency_us_p90": 75.60832891614, "counter_control_throughput_ratio": 0.9970138758468989, "counter_control_latency_ratio": 1.0175132705326957 }, "correctness": { "max_abs_error": 7.116789644580346e-11, "atol": 2e-06, "rtol": 2e-05 }, "throughput_us_p50": 56.7273901345, "throughput_us_p90": 57.7917587444, "latency_us_p50": 287.583, "latency_us_p90": 357.7586 }, "slowdown": 0.43559263955475075, "device_compute_slowdown": 0.415446482440776, "valid": true, "round": 3, "variant": "reference" } ] }