From c408c7231a5a040e609ca962671090c442ef40f5 Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:01:17 +0100 Subject: [PATCH 01/11] Add get_traces utility and adapt snakemake rules --- kernels/alloc/Snakefile | 8 +++++--- kernels/gemm/Snakefile | 8 +++++--- kernels/rescale/Snakefile | 8 +++++--- kernels/simple_copy/Snakefile | 8 +++++--- kernels/streamer_alu/Snakefile | 12 ++++++------ kernels/streamer_matmul/Snakefile | 10 +++++----- kernels/tiled_add/Snakefile | 21 ++++++++++++--------- util/snake/configs.py | 6 ++++++ util/snake/default_rules.smk | 20 ++++++++++++++++++++ util/snake/paths.py | 29 +++++++++++++++++++++++++++++ 10 files changed, 98 insertions(+), 32 deletions(-) diff --git a/kernels/alloc/Snakefile b/kernels/alloc/Snakefile index 2ef7276c..94b17f49 100644 --- a/kernels/alloc/Snakefile +++ b/kernels/alloc/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_mac_config +from util.snake.paths import get_traces config = get_snax_mac_config() @@ -28,11 +29,12 @@ module default_rules: config +files = ["func"] + + rule all: input: - "func.x", - shell: - "{config[vltsim]} {input[0]}" + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), use rule * from default_rules exclude compile_simple_main as default_* diff --git a/kernels/gemm/Snakefile b/kernels/gemm/Snakefile index 8fac9119..d3befaa6 100644 --- a/kernels/gemm/Snakefile +++ b/kernels/gemm/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_gemmx_config +from util.snake.paths import get_traces config = get_snax_gemmx_config() @@ -49,11 +50,12 @@ rule compile_main: "{config[cc]} {config[cflags]} -c {input[0]}" +files = ["gemm"] + + rule all: input: - "gemm.x", - shell: - "{config[vltsim]} {input[0]}" + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), from gendata import create_data_files diff --git a/kernels/rescale/Snakefile b/kernels/rescale/Snakefile index a87c7eb7..ddb67c5e 100644 --- a/kernels/rescale/Snakefile +++ b/kernels/rescale/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_gemmx_config +from util.snake.paths import get_traces config = get_snax_gemmx_config() config["snaxoptflags"] = ",".join( @@ -44,11 +45,12 @@ rule link_snax_binary: "{config[ld]} {config[ldflags]} {input} -o {output}" +files = ["rescale"] + + rule all: input: - "rescale.x", - shell: - "{config[vltsim]} {input[0]}" + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), from gendata import create_data_files diff --git a/kernels/simple_copy/Snakefile b/kernels/simple_copy/Snakefile index 85812aa1..a3474c32 100644 --- a/kernels/simple_copy/Snakefile +++ b/kernels/simple_copy/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_mac_config +from util.snake.paths import get_traces config = get_snax_mac_config() config["snaxoptflags"] = ",".join( @@ -29,12 +30,13 @@ module default_rules: use rule * from default_rules as default_* +files = ["simple_copy"] + + # Rules rule all: input: - "simple_copy.x", - shell: - "{config[vltsim]} {input[0]}" + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), rule compile_main: diff --git a/kernels/streamer_alu/Snakefile b/kernels/streamer_alu/Snakefile index d0edb627..c92d56dc 100644 --- a/kernels/streamer_alu/Snakefile +++ b/kernels/streamer_alu/Snakefile @@ -1,4 +1,6 @@ from util.snake.configs import get_snax_alu_config +from util.snake.paths import get_traces, get_trace_ext +import itertools config = get_snax_alu_config() config["snaxoptflags"] = ",".join( @@ -39,14 +41,12 @@ module default_rules: use rule * from default_rules as default_* -# Rules +files = ["streamer_add", "streamer_add_stream"] + + rule all: input: - "streamer_add.x", - "streamer_add_stream.x", - run: - for item in input: - shell("{config[vltsim]} {item}") + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), from gendata import create_data_files diff --git a/kernels/streamer_matmul/Snakefile b/kernels/streamer_matmul/Snakefile index 8ad6f9d7..c1a4f4b7 100644 --- a/kernels/streamer_matmul/Snakefile +++ b/kernels/streamer_matmul/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_gemmx_config +from util.snake.paths import get_traces config = get_snax_gemmx_config() config["snaxoptflags"] = ",".join( @@ -42,14 +43,13 @@ module default_rules: use rule * from default_rules as default_* +files = ["quantized_matmul", "tiled_quantized_matmul"] + + # Rules rule all: input: - "quantized_matmul.x", - "tiled_quantized_matmul.x", - run: - for item in input: - shell("{config[vltsim]} {item}") + *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), rule generate_quantized_matmul: diff --git a/kernels/tiled_add/Snakefile b/kernels/tiled_add/Snakefile index ab7fda5c..5f2682c3 100644 --- a/kernels/tiled_add/Snakefile +++ b/kernels/tiled_add/Snakefile @@ -1,4 +1,5 @@ from util.snake.configs import get_snax_alu_config +from util.snake.paths import get_traces config = get_snax_alu_config() @@ -122,16 +123,18 @@ COMPILER_OPTS = ["accfgboth", "noaccfgopt"] rule all: input: - expand( - "{file}_{array_size}_{tile_size}_{compiler_opt}.x", - file=FILES, - array_size=ARRAY_SIZES, - tile_size=TILE_SIZES, - compiler_opt=COMPILER_OPTS, + *get_traces( + expand( + "{file}_{array_size}_{tile_size}_{compiler_opt}", + file=FILES, + array_size=ARRAY_SIZES, + tile_size=TILE_SIZES, + compiler_opt=COMPILER_OPTS, + ), + config["num_chips"], + config["num_harts"], + "dasm", ), - run: - for item in input: - shell("{config[vltsim]} {item}") from gendata import generate_data diff --git a/util/snake/configs.py b/util/snake/configs.py index bbe9e86a..1d4f61e7 100644 --- a/util/snake/configs.py +++ b/util/snake/configs.py @@ -14,6 +14,8 @@ def get_snax_mac_config(): config = {} config.update(get_default_paths()) config.update(get_default_flags(snitch_sw_path)) + config["num_chips"] = 1 + config["num_harts"] = 2 config["vltsim"] = f"{snax_utils_path}/snax-mac-rtl/bin/snitch_cluster.vlt" config["cflags"].append( f"-I{snitch_sw_path}/target/snitch_cluster/sw/snax/mac/include" @@ -31,6 +33,8 @@ def get_snax_gemmx_config(): config = {} config.update(get_default_paths()) config.update(get_default_flags(snitch_sw_path)) + config["num_chips"] = 1 + config["num_harts"] = 2 config["vltsim"] = ( snax_utils_path + "/snax-kul-cluster-mixed-narrow-wide-rtl/bin/snitch_cluster.vlt" @@ -45,6 +49,8 @@ def get_snax_alu_config(): config = {} config.update(get_default_paths()) config.update(get_default_flags(snitch_sw_path)) + config["num_chips"] = 1 + config["num_harts"] = 2 config["vltsim"] = snax_utils_path + "/snax-alu-rtl/bin/snitch_cluster.vlt" return config diff --git a/util/snake/default_rules.smk b/util/snake/default_rules.smk index 4e8c827d..d70e4cf3 100644 --- a/util/snake/default_rules.smk +++ b/util/snake/default_rules.smk @@ -1,3 +1,6 @@ +from util.snake.paths import get_trace_ext + + rule snax_opt_mlir: """ Apply various transformations snax-opt on mlir files. @@ -54,3 +57,20 @@ rule clean: """ shell: "rm -rf *.ll12 *.x *.o *.logs/ logs/ data* *.dasm" + + +rule simulate: + input: + "{file}.x", + output: + temp( + multiext( + *get_trace_ext( + "{file}", config["num_chips"], config["num_harts"], "dasm" + ) + ) + ), + log: + "{file}.vltlog", + shell: + "{config[vltsim]} --prefix-trace={wildcards.file}_ {wildcards.file}.x 2>&1 | tee {log}" diff --git a/util/snake/paths.py b/util/snake/paths.py index 4f65677f..6d4f5049 100644 --- a/util/snake/paths.py +++ b/util/snake/paths.py @@ -1,3 +1,6 @@ +from collections.abc import Generator, Sequence + + def get_default_paths(): return { "cc": "clang", @@ -6,3 +9,29 @@ def get_default_paths(): "mlir-translate": "mlir-translate", "snax-opt": "snax-opt", } + + +def get_traces( + prefixes: Sequence[str], + num_chips: int = 1, + num_harts: int = 2, + extension: str = "dasm", +) -> Generator[str, None, None]: + for prefix in prefixes: + for chip_id in range(num_chips): + for hart_id in range(num_harts): + yield f"{prefix}_trace_chip_{chip_id:02d}_hart_{hart_id:05d}.{extension}" + + +def get_trace_ext( + prefix: str, num_chips: int = 1, num_harts: int = 2, extension: str = "dasm" +) -> Sequence[str]: + extensions = [ + trace[: 26 + len(extension)] + for trace in get_traces([""], num_chips, num_harts, extension) + ] + return [prefix, *extensions] + + +if __name__ == "__main__": + print(get_trace_ext("file", 1, 2, "dasm")) From 7425f14016edae68d5c73aa7392756b2bf68440b Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:05:39 +0100 Subject: [PATCH 02/11] Remove unused import --- kernels/streamer_alu/Snakefile | 1 - 1 file changed, 1 deletion(-) diff --git a/kernels/streamer_alu/Snakefile b/kernels/streamer_alu/Snakefile index c92d56dc..5b640962 100644 --- a/kernels/streamer_alu/Snakefile +++ b/kernels/streamer_alu/Snakefile @@ -1,6 +1,5 @@ from util.snake.configs import get_snax_alu_config from util.snake.paths import get_traces, get_trace_ext -import itertools config = get_snax_alu_config() config["snaxoptflags"] = ",".join( From 0b5c72d735bdadb093fe89dc28f83e68e4121ac5 Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:38:00 +0100 Subject: [PATCH 03/11] Use * operator instead of multiext --- util/snake/default_rules.smk | 13 ++++++------- util/snake/paths.py | 14 -------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/util/snake/default_rules.smk b/util/snake/default_rules.smk index d70e4cf3..3efa6699 100644 --- a/util/snake/default_rules.smk +++ b/util/snake/default_rules.smk @@ -1,4 +1,4 @@ -from util.snake.paths import get_trace_ext +from util.snake.paths import get_traces rule snax_opt_mlir: @@ -63,13 +63,12 @@ rule simulate: input: "{file}.x", output: - temp( - multiext( - *get_trace_ext( - "{file}", config["num_chips"], config["num_harts"], "dasm" - ) + *[ + temp(trace) + for trace in get_traces( + ["{file}"], config["num_chips"], config["num_harts"], "dasm" ) - ), + ], log: "{file}.vltlog", shell: diff --git a/util/snake/paths.py b/util/snake/paths.py index 6d4f5049..146f6b72 100644 --- a/util/snake/paths.py +++ b/util/snake/paths.py @@ -21,17 +21,3 @@ def get_traces( for chip_id in range(num_chips): for hart_id in range(num_harts): yield f"{prefix}_trace_chip_{chip_id:02d}_hart_{hart_id:05d}.{extension}" - - -def get_trace_ext( - prefix: str, num_chips: int = 1, num_harts: int = 2, extension: str = "dasm" -) -> Sequence[str]: - extensions = [ - trace[: 26 + len(extension)] - for trace in get_traces([""], num_chips, num_harts, extension) - ] - return [prefix, *extensions] - - -if __name__ == "__main__": - print(get_trace_ext("file", 1, 2, "dasm")) From da20093ce4b10560fd8a66b8b44699dc42ad0367 Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:39:45 +0100 Subject: [PATCH 04/11] Update lockfile --- pixi.lock | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/pixi.lock b/pixi.lock index d444dfec..12f09d44 100644 --- a/pixi.lock +++ b/pixi.lock @@ -39,9 +39,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.12-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.44-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyh707e725_8.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.10-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/immutables-0.21-py312h66e93f0_0.conda @@ -71,7 +71,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.6.3-hb9d3cd8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.28-pthreads_h94d23a6_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.48.0-hee588c1_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.48.0-hee588c1_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-hc0a3c3a_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-14.2.0-h4852527_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda @@ -140,7 +140,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9f/c9/02a1109cb6fa7ccf92c4384072fb25a2b58bcedead4052954d071ce97e37/filecheck-1.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/ec/00d68c4ddfedfe64159999e5f8a98fb8442729a63e2077eb9dcd89623d27/filelock-3.17.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/75/ed93520bceceabd78305717b3500224625f92e03617ba1f8e08d60e8eefb/flatbuffers-25.1.21-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/fa/1d103fe6e9bf174afd1c04772ca4f88e8f577f44d37b7cc8644fe5ff2620/fonttools-4.55.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/50/8a/e3c99a9aa1f9153fddf98c020449acc98140cd492d6264f8800fbd189c40/fonttools-4.55.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f3/cb/5c47b82fd1baf43dba973ae399095d51aaf0085ab0439838b4cbb1e87e3c/grpcio-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -665,10 +665,10 @@ packages: name: flatbuffers version: 25.1.21 sha256: 0e9736098ba8f4e48246a0640390f4992c0b1a734e7322a9463d5c3eea00558b -- pypi: https://files.pythonhosted.org/packages/7b/fa/1d103fe6e9bf174afd1c04772ca4f88e8f577f44d37b7cc8644fe5ff2620/fonttools-4.55.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl +- pypi: https://files.pythonhosted.org/packages/50/8a/e3c99a9aa1f9153fddf98c020449acc98140cd492d6264f8800fbd189c40/fonttools-4.55.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl name: fonttools - version: 4.55.4 - sha256: a2d0ac8656ada8b604ae5da15d9aa075232f2181b95b51a3a2a55195222df7e7 + version: 4.55.5 + sha256: e1a06e77f956a5857a4444f689f0ca4a1c4bdbcb38c812805f54a1b21380cc24 requires_dist: - fs>=2.2.0,<3 ; extra == 'ufo' - lxml>=4.0 ; extra == 'lxml' @@ -764,17 +764,17 @@ packages: requires_dist: - numpy>=1.19.3 requires_python: '>=3.9' -- conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyhd8ed1ab_1.conda - sha256: ec89b7e5b8aa2f0219f666084446e1fb7b54545861e9caa892acb24d125761b5 - md5: 2aa5ff7fa34a81b9196532c84c10d865 +- conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda + sha256: 6ad78a180576c706aabeb5b4c8ceb97c0cb25f1e112d76495bff23e3779948ba + md5: 0a802cb9888dd14eeefc611f05c40b6e depends: - python >=3.9 license: MIT license_family: MIT purls: - pkg:pypi/hpack?source=hash-mapping - size: 29412 - timestamp: 1733299296857 + size: 30731 + timestamp: 1737618390337 - conda: https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyh707e725_8.conda sha256: fa2071da7fab758c669e78227e6094f6b3608228740808a6de5d6bce83d9e52d md5: 7fe569c10905402ed47024fc481bb371 @@ -787,17 +787,17 @@ packages: - pkg:pypi/humanfriendly?source=hash-mapping size: 73563 timestamp: 1733928021866 -- conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_1.conda - sha256: e91c6ef09d076e1d9a02819cd00fa7ee18ecf30cdd667605c853980216584d1b - md5: 566e75c90c1d0c8c459eb0ad9833dc7a +- conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda + sha256: 77af6f5fe8b62ca07d09ac60127a30d9069fdc3c68d6b256754d0ffb1f7779f8 + md5: 8e6923fc12f1fe8f8c4e5c9f343256ac depends: - python >=3.9 license: MIT license_family: MIT purls: - pkg:pypi/hyperframe?source=hash-mapping - size: 17239 - timestamp: 1733298862681 + size: 17397 + timestamp: 1737618427549 - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda sha256: 71e750d509f5fa3421087ba88ef9a7b9be11c53174af3aa4d06aff4c18b38e8e md5: 8b189310083baabfb622af68fd9d3ae3 @@ -1208,6 +1208,8 @@ packages: depends: - __glibc >=2.17,<3.0.a0 - libgcc >=13 + constrains: + - xz ==5.6.3=*_1 arch: x86_64 platform: linux license: 0BSD @@ -1243,9 +1245,9 @@ packages: purls: [] size: 5578513 timestamp: 1730772671118 -- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.48.0-hee588c1_0.conda - sha256: 7bb84f44e1bd756da4a3d0d43308324a5533e6ba9f4772475884bce44d405064 - md5: 84bd1c9a82b455e7a2f390375fb38f90 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.48.0-hee588c1_1.conda + sha256: 22853d289ef6ec8a5b20f1aa261895b06525439990d3b139f8bfd0b5c5e32a3a + md5: 3fa05c528d8a1e2a67bbf1e36f22d3bc depends: - __glibc >=2.17,<3.0.a0 - libgcc >=13 @@ -1254,8 +1256,8 @@ packages: platform: linux license: Unlicense purls: [] - size: 876582 - timestamp: 1737123945341 + size: 878223 + timestamp: 1737564987837 - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-hc0a3c3a_1.conda sha256: 4661af0eb9bdcbb5fb33e5d0023b001ad4be828fccdcc56500059d56f9869462 md5: 234a5554c53625688d51062645337328 From ca8e5765a97bd532a48ed9af518c50d81bb3632d Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:56:58 +0100 Subject: [PATCH 05/11] Update lockfile --- pixi.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixi.lock b/pixi.lock index 12f09d44..d8dadf30 100644 --- a/pixi.lock +++ b/pixi.lock @@ -181,7 +181,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/89/9b/599bcfc7064fbe5740919e78c5df18e5dceb0887e676256a1061bb5ae232/virtualenv-20.29.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl - - pypi: git+https://github.com/xdslproject/xdsl.git@5c96c6efba255094871aabf1872e14ce706f9917 + - pypi: git+https://github.com/xdslproject/xdsl?rev=5c96c6efba255094871aabf1872e14ce706f9917#5c96c6efba255094871aabf1872e14ce706f9917 - pypi: . packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2467,7 +2467,7 @@ packages: - pkg:pypi/wrapt?source=hash-mapping size: 63590 timestamp: 1736869574299 -- pypi: git+https://github.com/xdslproject/xdsl.git@5c96c6efba255094871aabf1872e14ce706f9917 +- pypi: git+https://github.com/xdslproject/xdsl?rev=5c96c6efba255094871aabf1872e14ce706f9917#5c96c6efba255094871aabf1872e14ce706f9917 name: xdsl version: 0.26.0+21.g5c96c6ef requires_dist: From c287e8c089e969a7dc82fa03ced82d0b54c21b30 Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 14:58:56 +0100 Subject: [PATCH 06/11] Remove old import --- kernels/streamer_alu/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/streamer_alu/Snakefile b/kernels/streamer_alu/Snakefile index 5b640962..520bc35d 100644 --- a/kernels/streamer_alu/Snakefile +++ b/kernels/streamer_alu/Snakefile @@ -1,5 +1,5 @@ from util.snake.configs import get_snax_alu_config -from util.snake.paths import get_traces, get_trace_ext +from util.snake.paths import get_traces config = get_snax_alu_config() config["snaxoptflags"] = ",".join( From d913f7cf176aacc1ccb13d86bc690711e9d1c9be Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 15:16:55 +0100 Subject: [PATCH 07/11] Split up snax-specific rules --- kernels/alloc/Snakefile | 16 +++++++++++++--- kernels/gemm/Snakefile | 10 ++++++++++ kernels/rescale/Snakefile | 10 ++++++++++ kernels/simple_copy/Snakefile | 10 ++++++++++ kernels/streamer_alu/Snakefile | 10 ++++++++++ kernels/streamer_matmul/Snakefile | 10 ++++++++++ kernels/tiled_add/Snakefile | 10 ++++++++++ util/snake/default_rules.smk | 19 ------------------- util/snake/snax.smk | 17 +++++++++++++++++ 9 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 util/snake/snax.smk diff --git a/kernels/alloc/Snakefile b/kernels/alloc/Snakefile index 94b17f49..f6c2f31a 100644 --- a/kernels/alloc/Snakefile +++ b/kernels/alloc/Snakefile @@ -29,6 +29,19 @@ module default_rules: config +use rule * from default_rules as default_* + + +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + files = ["func"] @@ -37,9 +50,6 @@ rule all: *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), -use rule * from default_rules exclude compile_simple_main as default_* - - rule compile_snax_binary: input: "func.o", diff --git a/kernels/gemm/Snakefile b/kernels/gemm/Snakefile index d3befaa6..22d0bb0c 100644 --- a/kernels/gemm/Snakefile +++ b/kernels/gemm/Snakefile @@ -40,6 +40,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + rule compile_main: input: "main.c", diff --git a/kernels/rescale/Snakefile b/kernels/rescale/Snakefile index ddb67c5e..9e82a3dc 100644 --- a/kernels/rescale/Snakefile +++ b/kernels/rescale/Snakefile @@ -34,6 +34,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + rule link_snax_binary: input: "rescale.o", diff --git a/kernels/simple_copy/Snakefile b/kernels/simple_copy/Snakefile index a3474c32..9770d998 100644 --- a/kernels/simple_copy/Snakefile +++ b/kernels/simple_copy/Snakefile @@ -30,6 +30,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + files = ["simple_copy"] diff --git a/kernels/streamer_alu/Snakefile b/kernels/streamer_alu/Snakefile index 520bc35d..cd3e292e 100644 --- a/kernels/streamer_alu/Snakefile +++ b/kernels/streamer_alu/Snakefile @@ -40,6 +40,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + files = ["streamer_add", "streamer_add_stream"] diff --git a/kernels/streamer_matmul/Snakefile b/kernels/streamer_matmul/Snakefile index c1a4f4b7..20b9879c 100644 --- a/kernels/streamer_matmul/Snakefile +++ b/kernels/streamer_matmul/Snakefile @@ -40,9 +40,19 @@ module default_rules: config +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + use rule * from default_rules as default_* +use rule * from snax_rules as snax_* + + files = ["quantized_matmul", "tiled_quantized_matmul"] diff --git a/kernels/tiled_add/Snakefile b/kernels/tiled_add/Snakefile index 5f2682c3..010f740b 100644 --- a/kernels/tiled_add/Snakefile +++ b/kernels/tiled_add/Snakefile @@ -56,6 +56,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + rule size_mlir: input: "{file}.mlir", diff --git a/util/snake/default_rules.smk b/util/snake/default_rules.smk index 3efa6699..4e8c827d 100644 --- a/util/snake/default_rules.smk +++ b/util/snake/default_rules.smk @@ -1,6 +1,3 @@ -from util.snake.paths import get_traces - - rule snax_opt_mlir: """ Apply various transformations snax-opt on mlir files. @@ -57,19 +54,3 @@ rule clean: """ shell: "rm -rf *.ll12 *.x *.o *.logs/ logs/ data* *.dasm" - - -rule simulate: - input: - "{file}.x", - output: - *[ - temp(trace) - for trace in get_traces( - ["{file}"], config["num_chips"], config["num_harts"], "dasm" - ) - ], - log: - "{file}.vltlog", - shell: - "{config[vltsim]} --prefix-trace={wildcards.file}_ {wildcards.file}.x 2>&1 | tee {log}" diff --git a/util/snake/snax.smk b/util/snake/snax.smk new file mode 100644 index 00000000..d33cba72 --- /dev/null +++ b/util/snake/snax.smk @@ -0,0 +1,17 @@ +from util.snake.paths import get_traces + + +rule simulate: + input: + "{file}.x", + output: + *[ + temp(trace) + for trace in get_traces( + ["{file}"], config["num_chips"], config["num_harts"], "dasm" + ) + ], + log: + "{file}.vltlog", + shell: + "{config[vltsim]} --prefix-trace={wildcards.file}_ {wildcards.file}.x 2>&1 | tee {log}" From 6fd8b8fbe55e3eb335f05df6e37b35dbdc71b218 Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 18:12:52 +0100 Subject: [PATCH 08/11] Stop using get_traces --- kernels/alloc/Snakefile | 7 ++++++- kernels/gemm/Snakefile | 7 ++++++- kernels/rescale/Snakefile | 8 ++++++-- kernels/simple_copy/Snakefile | 8 ++++++-- kernels/streamer_alu/Snakefile | 8 ++++++-- kernels/streamer_matmul/Snakefile | 8 ++++++-- kernels/tiled_add/Snakefile | 20 ++++++++------------ kernels/transform_copy/Snakefile | 15 ++++++++------- util/snake/paths.py | 15 --------------- 9 files changed, 52 insertions(+), 44 deletions(-) diff --git a/kernels/alloc/Snakefile b/kernels/alloc/Snakefile index f6c2f31a..69b04b7f 100644 --- a/kernels/alloc/Snakefile +++ b/kernels/alloc/Snakefile @@ -47,7 +47,12 @@ files = ["func"] rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), rule compile_snax_binary: diff --git a/kernels/gemm/Snakefile b/kernels/gemm/Snakefile index 22d0bb0c..ee742a29 100644 --- a/kernels/gemm/Snakefile +++ b/kernels/gemm/Snakefile @@ -65,7 +65,12 @@ files = ["gemm"] rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), from gendata import create_data_files diff --git a/kernels/rescale/Snakefile b/kernels/rescale/Snakefile index 9e82a3dc..1615c173 100644 --- a/kernels/rescale/Snakefile +++ b/kernels/rescale/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_gemmx_config -from util.snake.paths import get_traces config = get_snax_gemmx_config() config["snaxoptflags"] = ",".join( @@ -60,7 +59,12 @@ files = ["rescale"] rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), from gendata import create_data_files diff --git a/kernels/simple_copy/Snakefile b/kernels/simple_copy/Snakefile index 9770d998..e40ef5fc 100644 --- a/kernels/simple_copy/Snakefile +++ b/kernels/simple_copy/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_mac_config -from util.snake.paths import get_traces config = get_snax_mac_config() config["snaxoptflags"] = ",".join( @@ -46,7 +45,12 @@ files = ["simple_copy"] # Rules rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), rule compile_main: diff --git a/kernels/streamer_alu/Snakefile b/kernels/streamer_alu/Snakefile index cd3e292e..e69c3e14 100644 --- a/kernels/streamer_alu/Snakefile +++ b/kernels/streamer_alu/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_alu_config -from util.snake.paths import get_traces config = get_snax_alu_config() config["snaxoptflags"] = ",".join( @@ -55,7 +54,12 @@ files = ["streamer_add", "streamer_add_stream"] rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), from gendata import create_data_files diff --git a/kernels/streamer_matmul/Snakefile b/kernels/streamer_matmul/Snakefile index 20b9879c..d5d235d8 100644 --- a/kernels/streamer_matmul/Snakefile +++ b/kernels/streamer_matmul/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_gemmx_config -from util.snake.paths import get_traces config = get_snax_gemmx_config() config["snaxoptflags"] = ",".join( @@ -59,7 +58,12 @@ files = ["quantized_matmul", "tiled_quantized_matmul"] # Rules rule all: input: - *get_traces(files, config["num_chips"], config["num_harts"], "dasm"), + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), rule generate_quantized_matmul: diff --git a/kernels/tiled_add/Snakefile b/kernels/tiled_add/Snakefile index 010f740b..395f57fb 100644 --- a/kernels/tiled_add/Snakefile +++ b/kernels/tiled_add/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_alu_config -from util.snake.paths import get_traces config = get_snax_alu_config() @@ -133,17 +132,14 @@ COMPILER_OPTS = ["accfgboth", "noaccfgopt"] rule all: input: - *get_traces( - expand( - "{file}_{array_size}_{tile_size}_{compiler_opt}", - file=FILES, - array_size=ARRAY_SIZES, - tile_size=TILE_SIZES, - compiler_opt=COMPILER_OPTS, - ), - config["num_chips"], - config["num_harts"], - "dasm", + expand( + "{file}_{array_size}_{tile_size}_{compiler_opt}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=FILES, + array_size=ARRAY_SIZES, + tile_size=TILE_SIZES, + compiler_opt=COMPILER_OPTS, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), ), diff --git a/kernels/transform_copy/Snakefile b/kernels/transform_copy/Snakefile index 6905ae32..ec6632cb 100644 --- a/kernels/transform_copy/Snakefile +++ b/kernels/transform_copy/Snakefile @@ -30,17 +30,18 @@ use rule * from default_rules as default_* from gendata import create_files +files = ["transform_copy", "transform_from_none", "transform_from_strided"] + # Rules rule all: input: - "transform_copy.x", - "transform_from_none.x", - "transform_from_strided.x", - run: - shell("{config[vltsim]} {input[0]} {input[0]}") - shell("{config[vltsim]} {input[1]} {input[1]}") - shell("{config[vltsim]} {input[2]} {input[2]}") + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=files, + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), rule generate_data: diff --git a/util/snake/paths.py b/util/snake/paths.py index 146f6b72..4f65677f 100644 --- a/util/snake/paths.py +++ b/util/snake/paths.py @@ -1,6 +1,3 @@ -from collections.abc import Generator, Sequence - - def get_default_paths(): return { "cc": "clang", @@ -9,15 +6,3 @@ def get_default_paths(): "mlir-translate": "mlir-translate", "snax-opt": "snax-opt", } - - -def get_traces( - prefixes: Sequence[str], - num_chips: int = 1, - num_harts: int = 2, - extension: str = "dasm", -) -> Generator[str, None, None]: - for prefix in prefixes: - for chip_id in range(num_chips): - for hart_id in range(num_harts): - yield f"{prefix}_trace_chip_{chip_id:02d}_hart_{hart_id:05d}.{extension}" From 8d36b7bf415853eb92bbee2ed3eeae534bd9d4fe Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 18:17:25 +0100 Subject: [PATCH 09/11] Update main simulation rule --- util/snake/snax.smk | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util/snake/snax.smk b/util/snake/snax.smk index d33cba72..09177998 100644 --- a/util/snake/snax.smk +++ b/util/snake/snax.smk @@ -1,16 +1,15 @@ -from util.snake.paths import get_traces - - rule simulate: input: "{file}.x", output: - *[ - temp(trace) - for trace in get_traces( - ["{file}"], config["num_chips"], config["num_harts"], "dasm" - ) - ], + temp( + expand( + "{file}_trace_chip_{num_chips:02d}_hart_{num_harts:05d}.dasm", + file=["{file}"], + num_chips=range(config["num_chips"]), + num_harts=range(config["num_harts"]), + ), + ), log: "{file}.vltlog", shell: From da8ba61c8b799df62066bb4b8c5289b8aba30dcd Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 18:19:31 +0100 Subject: [PATCH 10/11] Add snax module in transform_copy snakefile --- kernels/transform_copy/Snakefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernels/transform_copy/Snakefile b/kernels/transform_copy/Snakefile index ec6632cb..c98b518c 100644 --- a/kernels/transform_copy/Snakefile +++ b/kernels/transform_copy/Snakefile @@ -28,6 +28,16 @@ module default_rules: use rule * from default_rules as default_* +module snax_rules: + snakefile: + "../../util/snake/snax.smk" + config: + config + + +use rule * from snax_rules as snax_* + + from gendata import create_files files = ["transform_copy", "transform_from_none", "transform_from_strided"] From c68b45ca9e7268419bb69d43263fdc3d7aaadbdc Mon Sep 17 00:00:00 2001 From: Josse Van Delm Date: Thu, 23 Jan 2025 18:21:01 +0100 Subject: [PATCH 11/11] Remove unnecessary imports --- kernels/alloc/Snakefile | 1 - kernels/gemm/Snakefile | 1 - 2 files changed, 2 deletions(-) diff --git a/kernels/alloc/Snakefile b/kernels/alloc/Snakefile index 69b04b7f..6b4ce2c7 100644 --- a/kernels/alloc/Snakefile +++ b/kernels/alloc/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_mac_config -from util.snake.paths import get_traces config = get_snax_mac_config() diff --git a/kernels/gemm/Snakefile b/kernels/gemm/Snakefile index ee742a29..d8236bc4 100644 --- a/kernels/gemm/Snakefile +++ b/kernels/gemm/Snakefile @@ -1,5 +1,4 @@ from util.snake.configs import get_snax_gemmx_config -from util.snake.paths import get_traces config = get_snax_gemmx_config()