diff --git a/CMakeLists.txt b/CMakeLists.txt index aae997fcb21..d5f8cd6bc47 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -362,7 +362,7 @@ rocm_package_add_rpm_dependencies(SHARED_DEPENDS "hip-devel") rocm_create_package( NAME MIGraphX - DESCRIPTION "AMD's graph optimizer" + DESCRIPTION "AMD graph optimizer" MAINTAINER "AMDMIGraphX Maintainer " LDCONFIG PTH diff --git a/src/include/migraphx/op/reshape.hpp b/src/include/migraphx/op/reshape.hpp index d87bb4e82e0..18824a7d8ba 100644 --- a/src/include/migraphx/op/reshape.hpp +++ b/src/include/migraphx/op/reshape.hpp @@ -76,22 +76,24 @@ struct reshape const bool has_negative_dim_attr = neg_dim_num < dims.size(); // construct output dynamic shape from dims attribute std::vector output_dyn_dims(dims.size()); - std::transform(dims.begin(), - dims.end(), - input_dyn_dims.begin(), - output_dyn_dims.begin(), - [](auto dim, auto input_dyn_dim) -> shape::dynamic_dimension { - if(dim == 0) - { - return input_dyn_dim; - } - if(dim == -1) - { - return {1, 1}; - } - std::size_t u_dim = dim; - return {u_dim, u_dim}; - }); + // NOTE: input_dyn_dims.size() may not equal dims.size() + for(std::size_t i = 0; i < dims.size(); ++i) + { + auto d = dims.at(i); + if(d == 0) + { + output_dyn_dims.at(i) = input_dyn_dims.at(i); + } + else if(d == -1) + { + output_dyn_dims.at(i) = {1, 1}; + } + else + { + std::size_t u_dim = d; + output_dyn_dims.at(i) = {u_dim, u_dim}; + } + } if(has_negative_dim_attr) { diff --git a/src/targets/gpu/lowering.cpp b/src/targets/gpu/lowering.cpp index efce0fd570e..2756231b524 100644 --- a/src/targets/gpu/lowering.cpp +++ b/src/targets/gpu/lowering.cpp @@ -36,6 +36,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,7 @@ struct miopen_apply add_extend_op("rnn_var_sl_shift_sequence"); add_extend_op("topk"); add_generic_op("contiguous"); + add_pooling_op(); #if MIGRAPHX_USE_MIOPEN add_convolution_op("convolution"); add_convolution_op("convolution_backwards"); @@ -296,6 +298,30 @@ struct miopen_apply }); } + void add_pooling_op() + { + apply_map.emplace("pooling", [=](instruction_ref ins) { + auto&& op = ins->get_operator(); + auto op_val = op.to_value(); + if(op_val.at("count_include_pad").to()) + { + return insert_precompile_op(ins); + } + if(op_val["mode"].to() == op::pooling_mode::lpnorm) + { + return insert_precompile_op(ins); + } +#if MIGRAPHX_USE_MIOPEN + auto output = insert_allocation(ins, ins->get_shape()); + std::vector refs = ins->inputs(); + refs.push_back(output); + return mod->replace_instruction(ins, make_op("gpu::pooling", op.to_value()), refs); +#else + return insert_precompile_op(ins); +#endif + }); + } + // use 0 - input to represent neg void add_neg_op() { diff --git a/test/op_shape_test.cpp b/test/op_shape_test.cpp index 61918317fe0..63f1935094a 100644 --- a/test/op_shape_test.cpp +++ b/test/op_shape_test.cpp @@ -3270,6 +3270,22 @@ TEST_CASE(reshape_dyn_1in_negative_1_dims_1) expect_shape(output, migraphx::make_op("reshape", {{"dims", {0, -1, 2, 2}}}), input); } +TEST_CASE(reshape_dyn_1in_negative_1_dims_2) +{ + migraphx::shape input{migraphx::shape::float_type, {{1, 4}, {24, 24}, {2, 8}, {2, 8}}}; + std::vector out_dyn_dims = {{1, 4}, {24, 24}, {4, 64}}; + migraphx::shape output{migraphx::shape::float_type, out_dyn_dims}; + expect_shape(output, migraphx::make_op("reshape", {{"dims", {0, 0, -1}}}), input); +} + +TEST_CASE(reshape_dyn_1in_negative_1_dims_3) +{ + migraphx::shape input{migraphx::shape::float_type, {{1, 4}, {24, 24}}}; + std::vector out_dyn_dims = {{1, 4}, {4, 4}, {3, 3}, {2, 2}}; + migraphx::shape output{migraphx::shape::float_type, out_dyn_dims}; + expect_shape(output, migraphx::make_op("reshape", {{"dims", {0, 4, 3, 2}}}), input); +} + // note how non-fixed dynamic dimension on axis=0 goes to 2 from `dims` attribute // code assumes that this will work at run-time TEST_CASE(reshape_dyn_1in_dyn_to_fixed) diff --git a/tools/model_zoo/README.md b/tools/model_zoo/README.md new file mode 100644 index 00000000000..e035f889cd6 --- /dev/null +++ b/tools/model_zoo/README.md @@ -0,0 +1,4 @@ +# Model Zoo + +- [Test Generator with Datasets](./test_generator/) +- [ONNX Zoo](./onnx_zoo/) diff --git a/tools/model_zoo/onnx_zoo/README.md b/tools/model_zoo/onnx_zoo/README.md new file mode 100644 index 00000000000..ed861bb8a56 --- /dev/null +++ b/tools/model_zoo/onnx_zoo/README.md @@ -0,0 +1,50 @@ +# ONNX Zoo model tester + +Helper script to test [`ONNX Zoo models`](https://onnx.ai/models/) which have test data with [`test_runner.py`](../../test_runner.py) + +## Getting the repository + +> [!IMPORTANT] +> Make sure to enable git-lfs. + +```bash +git clone https://github.com/onnx/models.git --depth 1 +``` + +## Running the tests + +> [!IMPORTANT] +> The argument must point to a folder, not a file. + +```bash +# VERBOSE=1 DEBUG=1 # use these for more log +# ATOL=0.001 RTOL=0.001 TARGET=gpu # are the default values +./test_models.sh models/validated +``` + +You can also pass multiple folders, e.g.: + +```bash +./test_models.sh models/validated/text/machine_comprehension/t5/ models/validated/vision/classification/shufflenet/ +``` + +## Results + +Result are separated by dtype: `logs/fp32` and `logs/fp16` + +### Helpers + +```bash +# Something went wrong +grep -HRL PASSED logs +# Runtime error +grep -HRi RuntimeError logs/ +# Accuracy issue +grep -HRl FAILED logs +``` + +## Cleanup + +If at any point something fails, the following things might need cleanup: +- Remove `tmp_model` folder +- `git lfs prune` in `models` \ No newline at end of file diff --git a/tools/model_zoo/onnx_zoo/test_models.sh b/tools/model_zoo/onnx_zoo/test_models.sh new file mode 100755 index 00000000000..84ff0477382 --- /dev/null +++ b/tools/model_zoo/onnx_zoo/test_models.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### + +set -e + +WORK_DIR="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_PATH=$(dirname $(dirname $(dirname $(readlink -f "$0"))))/test_runner.py +TESTER_SCRIPT="${TESTER:-$SCRIPT_PATH}" +ATOL="${ATOL:-0.001}" +RTOL="${RTOL:-0.001}" +TARGET="${TARGET:-gpu}" + +if [[ "${DEBUG:-0}" -eq 1 ]]; then + PIPE=/dev/stdout +else + PIPE=/dev/null +fi + +if [[ "${VERBOSE:-0}" -eq 1 ]]; then + set -x +fi + +# Iterate through input recursively, process any tar.gz file +function iterate() { + local dir="$1" + + for file in "$dir"/*; do + if [ -f "$file" ]; then + if [[ $file = *.tar.gz ]]; then + process "$file" + fi + fi + + if [ -d "$file" ]; then + iterate "$file" + fi + done +} + +# Process will download the lfs file, extract model and test data +# Test it with test_runner.py, then cleanup +function process() { + local file="$1" + echo "INFO: process $file started" + setup $file + test $file fp32 + test $file fp16 + cleanup $file + echo "INFO: process $file finished" +} + +# Download and extract files +function setup() { + local file="$1" + echo "INFO: setup $file" + local_file="$(basename $file)" + # We need to change the folder to pull the file + folder="$(cd -P -- "$(dirname -- "$file")" && pwd -P)" + cd $folder &> "${PIPE}" && git lfs pull --include="$local_file" --exclude="" &> "${PIPE}"; cd - &> "${PIPE}" + tar xzf $file -C $WORK_DIR/tmp_model &> "${PIPE}" +} + +# Remove tmp files and prune models +function cleanup() { + local file="$1" + echo "INFO: cleanup $file" + # We need to change the folder to pull the file + folder="$(cd -P -- "$(dirname -- "$file")" && pwd -P)" + cd $folder &> "${PIPE}" && git lfs prune &> "${PIPE}"; cd - &> "${PIPE}" + rm -r $WORK_DIR/tmp_model/* &> "${PIPE}" +} + +# Run test_runner.py and log if something goes wrong +function test() { + local file="$1" + echo "INFO: test $file ($2)" + local_file="$(basename $file)" + flag="--atol $ATOL --rtol $RTOL --target $TARGET" + if [[ "$2" = "fp16" ]]; then + flag="$flag --fp16" + fi + EXIT_CODE=0 + python3 $TESTER_SCRIPT ${flag} $WORK_DIR/tmp_model/*/ &> "$WORK_DIR/logs/$2/${local_file//\//_}.log" || EXIT_CODE=$? + if [[ "${EXIT_CODE:-0}" -ne 0 ]]; then + echo "WARNING: ${file} failed ($2)" + fi +} + +mkdir -p $WORK_DIR/logs/fp32/ $WORK_DIR/logs/fp16/ $WORK_DIR/tmp_model +rm -fr $WORK_DIR/tmp_model/* + +for arg in "$@"; do + iterate "$(dirname $(readlink -e $arg))/$(basename $arg)" +done diff --git a/tools/model_zoo/test_generator/README.md b/tools/model_zoo/test_generator/README.md new file mode 100644 index 00000000000..1e3ce45b264 --- /dev/null +++ b/tools/model_zoo/test_generator/README.md @@ -0,0 +1,117 @@ +# Test Generator with Datasets + +Helper module to generate real samples from datasets for specific models. + +## Prerequisites + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r requirements.txt +``` + +To use audio based datasets, install sndfile +```bash +apt install libsndfile1 +``` + +## Usage + +```bash +usage: generate.py [-h] + [--image {all,none,...}] + [--text {all,none,...}] + [--audio {all,none,...}] + [--output-folder-prefix OUTPUT_FOLDER_PREFIX] + [--sample-limit SAMPLE_LIMIT] + [--decode-limit DECODE_LIMIT] + +optional arguments: + -h, --help show this help message and exit + --image {all,none,...} + Image models to test with imagenet-2012-val dataset samples + --text {all,none,...} + Text models to test with squad-hf dataset samples + --audio {all,none,...} + Audio models to test with librispeech-asr dataset samples + --output-folder-prefix OUTPUT_FOLDER_PREFIX + Output path will be "//" + --sample-limit SAMPLE_LIMIT + Max number of samples generated. Use 0 to ignore it. + --decode-limit DECODE_LIMIT + Max number of sum-samples generated for decoder models. Use 0 to ignore it. (Only for decoder models) +``` + +> [!NOTE] +> Some models require permission to access, use `huggingface-cli login`. + +To generate everything: +```bash +python generate.py +``` + +To generate a subset of the supported models: +- `none` to skip it +- `all` for every models +- list supported model names + +```bash +python generate.py --image resnet50_v1.5 clip-vit-large-patch14 --text none --audio none +``` + +## Test models + +`test_models.sh` will run all downloaded models on the `generated` samples. The result will be in `logs`. + +```bash +./test_models.sh generated/ +``` + +> [!NOTE] +> `generated` is the default output folder, make sure to match `--output-folder-prefix` name. + +## Adding more models + +To add mode models, first choose the proper place: +- [image](./sample_generator/model/image.py) +- [text](./sample_generator/model/text.py) +- [audio](./sample_generator/model/audio.py) +- [hybrid](./sample_generator/model/hybrid.py) + +For example, adding basic would be this (e.g. ResNet): + +```python +class ResNet50_v1_5(OptimumHFModelDownloadMixin, + AutoImageProcessorHFMixin, BaseModel): + @property + def model_id(self): + return "microsoft/resnet-50" + + @staticmethod + def name(): + return "resnet50_v1.5" +``` + +Define the class with the proper `Mixin`s: +- `OptimumHFModelDownloadMixin`: Download model from Hugging Face and export it to onnx with Optimum +- `AutoImageProcessorHFMixin`: Define the processor from Hugging Face (This depends on the model type) +- `BaseModel`: Default model type, other choice is `DecoderModel` + +Provide 2 mandatory fields: +- `model_id`: Hugging Face url +- `name`: unique name for model + +To add a more complex model (e.g. Decoder), check [text](./sample_generator/model/text.py). + +The [generate](./generate.py) part will need further updating to include the model. + +## Adding more datasets + +The 3 most common use cases are handled: +- `Image`: with [imagenet](./sample_generator/dataset/imagenet.py) +- `Text`: with [squad](./sample_generator/dataset/squad.py) +- `Audio`: with [librispeech](./sample_generator/dataset/librispeech.py) + +To add a new use case, e.g. Video, create a new python file in dataset, and inherit a new class from Base. + +The [generate](./generate.py) part will need further updating to include the dataset. diff --git a/tools/model_zoo/test_generator/generate.py b/tools/model_zoo/test_generator/generate.py new file mode 100644 index 00000000000..3c2af3ae58d --- /dev/null +++ b/tools/model_zoo/test_generator/generate.py @@ -0,0 +1,211 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from argparse import ArgumentParser +from collections import namedtuple +import warnings + +warnings.filterwarnings('ignore') + +# datasets +from sample_generator.dataset.imagenet import ImageNet2012Val +from sample_generator.dataset.coco import COCO2017Val +from sample_generator.dataset.librispeech import LibriSpeechASR +from sample_generator.dataset.squad import SQuAD_HF +from sample_generator.dataset.prompts import StylePrompts + +# models +from sample_generator.model.image import ResNet50_v1, ResNet50_v1_5, VitBasePatch16_224, TIMM_MobileNetv3_large +from sample_generator.model.text import BERT_large_uncased, DistilBERT_base_cased_distilled_SQuAD, RobertaBaseSquad2 +from sample_generator.model.text import GPTJ, Llama2_7b_chat_hf, Llama3_8b_instruct, T5_base, Gemma_2b_it +from sample_generator.model.audio import Wav2Vec2_base_960h, WhisperSmallEn +from sample_generator.model.hybrid import ClipVitLargePatch14 +from sample_generator.model.diffusion import StableDiffusion21, StableDiffusionXL + +# generator +from sample_generator.generator import generate_test_dataset, generate_diffusion_data + +DatasetModelsPair = namedtuple('DatasetModelsPair', ['dataset', 'models']) + +imagenet_models = ( + ResNet50_v1, + ResNet50_v1_5, + VitBasePatch16_224, + TIMM_MobileNetv3_large, + ClipVitLargePatch14, +) + +squad_models = ( + BERT_large_uncased, + DistilBERT_base_cased_distilled_SQuAD, + RobertaBaseSquad2, + GPTJ, + T5_base, + Gemma_2b_it, + Llama2_7b_chat_hf, + Llama3_8b_instruct, +) + +librispeech_models = (Wav2Vec2_base_960h, WhisperSmallEn) + +diffusion_models = (StableDiffusion21, StableDiffusionXL) + +default_dataset_model_mapping = { + "image": DatasetModelsPair(ImageNet2012Val, imagenet_models), + "text": DatasetModelsPair(SQuAD_HF, squad_models), + "audio": DatasetModelsPair(LibriSpeechASR, librispeech_models), + "diffusion": DatasetModelsPair((COCO2017Val, StylePrompts), + diffusion_models) +} + + +def get_args(): + parser = ArgumentParser() + parser.add_argument( + '--image', + choices=['all', 'none'] + [model.name() for model in imagenet_models], + nargs='+', + default='all', + dest='image_model_names', + type=str, + help= + f'Image models to test with {ImageNet2012Val.name()} dataset samples') + parser.add_argument( + '--text', + choices=['all', 'none'] + [model.name() for model in squad_models], + nargs='+', + default='all', + dest='text_model_names', + type=str, + help=f'Text models to test with {SQuAD_HF.name()} dataset samples') + parser.add_argument( + '--audio', + choices=['all', 'none'] + + [model.name() for model in librispeech_models], + nargs='+', + default='all', + dest='audio_model_names', + type=str, + help= + f'Audio models to test with {LibriSpeechASR.name()} dataset samples') + parser.add_argument( + '--diffusion', + choices=['all', 'none'] + [model.name() for model in diffusion_models], + nargs='+', + default='all', + dest='diffusion_model_names', + type=str, + help= + f'DiffusionModel models to test with {COCO2017Val.name()} and {StylePrompts} dataset samples' + ) + parser.add_argument( + '--output-folder-prefix', + default='generated', + help='Output path will be "//"') + parser.add_argument( + '--sample-limit', + default=5, + type=int, + help="Max number of samples generated. Use 0 to ignore it.") + parser.add_argument( + '--decode-limit', + default=5, + type=int, + help= + "Max number of sum-samples generated for decoder models. Use 0 to ignore it. (Only for decoder models)" + ) + return parser.parse_args() + + +def get_dataset_models_pairs(dataset_type, model_names): + if 'none' in model_names: + return None + ds_ms_mapping = default_dataset_model_mapping[dataset_type] + if 'all' in model_names: + return ds_ms_mapping + + return DatasetModelsPair(dataset=ds_ms_mapping.dataset, + models=(model for model in ds_ms_mapping.models + if model.name() in model_names)) + + +def main(image_model_names='all', + text_model_names='all', + audio_model_names='all', + diffusion_model_names='all', + output_folder_prefix='generated', + sample_limit=5, + decode_limit=5): + for dataset_type, model_names in zip( + ('image', 'text', 'audio'), + (image_model_names, text_model_names, audio_model_names)): + dataset_model_pair = get_dataset_models_pairs(dataset_type, + model_names) + if dataset_model_pair is None: + print(f"Skip {dataset_type}...") + continue + + dataset = dataset_model_pair.dataset + for model in dataset_model_pair.models: + try: + generate_test_dataset( + model(), + dataset(), + output_folder_prefix=output_folder_prefix, + sample_limit=sample_limit, + decode_limit=decode_limit) + except Exception as e: + print(f"Something went wrong:\n{e}\nSkipping model...") + continue + + for dataset_type, model_names in zip(('diffusion', ), + (diffusion_model_names)): + dataset_model_pair = get_dataset_models_pairs(dataset_type, + model_names) + if dataset_model_pair is None: + print(f"Skip {dataset_type}...") + continue + + image_dataset, prompt_dataset = dataset_model_pair.dataset + for model in dataset_model_pair.models: + try: + generate_diffusion_data( + model(), + image_dataset(), + prompt_dataset(), + output_folder_prefix=output_folder_prefix, + sample_limit=sample_limit, + decode_limit=decode_limit) + except Exception as e: + print(f"Something went wrong:\n{e}\nSkipping model...") + continue + + print( + f'Use this to test MIGraphX with the result:\n./test_models.sh {output_folder_prefix}' + ) + + +if '__main__' == __name__: + args = get_args() + main(**vars(args)) diff --git a/tools/model_zoo/test_generator/requirements.txt b/tools/model_zoo/test_generator/requirements.txt new file mode 100644 index 00000000000..2b6a19f57fd --- /dev/null +++ b/tools/model_zoo/test_generator/requirements.txt @@ -0,0 +1,41 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +-f https://repo.radeon.com/rocm/manylinux/rocm-rel-6.1/ +torch == 2.1.2 +torchvision == 0.16.1 +optimum >= 1.19.0 +accelerate +datasets +numpy == 1.24.1 +onnx +huggingface_hub +Pillow +opencv-python-headless +onnxruntime +transformers +diffusers +timm +soundfile +librosa diff --git a/tools/model_zoo/test_generator/sample_generator/__init__.py b/tools/model_zoo/test_generator/sample_generator/__init__.py new file mode 100644 index 00000000000..508b4b8c9de --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/__init__.py @@ -0,0 +1,25 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +__all__ = ["dataset", "generator", "model", "utils"] diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/__init__.py b/tools/model_zoo/test_generator/sample_generator/dataset/__init__.py new file mode 100644 index 00000000000..201932117cd --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/__init__.py @@ -0,0 +1,25 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +__all__ = ["imagenet", "coco", "librispeech", "prompts", "squad"] diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/base.py b/tools/model_zoo/test_generator/sample_generator/dataset/base.py new file mode 100644 index 00000000000..085fbdf9a12 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/base.py @@ -0,0 +1,66 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import abc +from datasets import load_dataset + + +class BaseDataset(abc.ABC): + @property + @abc.abstractmethod + def url(self): + pass + + @property + @abc.abstractmethod + def split(self): + pass + + @staticmethod + @abc.abstractmethod + def name(): + pass + + @abc.abstractmethod + def __iter__(self): + pass + + @abc.abstractmethod + def __next__(self): + pass + + @abc.abstractmethod + def transform(self, *args, **kwargs): + pass + + +class ValidationDatasetHFIteratorMixin(object): + def __iter__(self): + print(f"Load dataset from {self.url} using {self.split} split") + self.dataset = iter( + load_dataset(self.url, split=self.split, streaming=True)) + return self.dataset + + def __next__(self): + return next(self.dataset) diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/coco.py b/tools/model_zoo/test_generator/sample_generator/dataset/coco.py new file mode 100644 index 00000000000..5c47b89c1d8 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/coco.py @@ -0,0 +1,46 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseDataset, ValidationDatasetHFIteratorMixin + + +class COCO2017Val(ValidationDatasetHFIteratorMixin, BaseDataset): + @property + def url(self): + return "lmms-lab/COCO-Caption2017" + + @property + def split(self): + return "val" + + @staticmethod + def name(): + return "coco-2017-val" + + def transform(self, inputs, data, prepocess_fn): + result = prepocess_fn(data["image"]) + inputs, keys = sorted(inputs), sorted(list(result.keys())) + assert inputs == keys, f"{inputs = } == {keys = }" + # The result should be a simple dict, the preproc returns a wrapped class, dict() will remove it + return dict(result) diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/imagenet.py b/tools/model_zoo/test_generator/sample_generator/dataset/imagenet.py new file mode 100644 index 00000000000..62c3a86d05d --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/imagenet.py @@ -0,0 +1,59 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseDataset +from datasets import load_dataset + + +class ImageNet2012Val(BaseDataset): + @property + def url(self): + return "https://image-net.org/data/ILSVRC/2012/ILSVRC2012_img_val.tar" + + @property + def split(self): + return "val" + + @staticmethod + def name(): + return "imagenet-2012-val" + + def __iter__(self): + print(f"Load dataset from {self.url}") + self.dataset = iter( + load_dataset("webdataset", + data_files={self.split: self.url}, + split=self.split, + streaming=True)) + return self.dataset + + def __next__(self): + return next(self.dataset) + + def transform(self, inputs, data, prepocess_fn): + result = prepocess_fn(data["jpeg"]) + inputs, keys = sorted(inputs), sorted(list(result.keys())) + assert inputs == keys, f"{inputs = } == {keys = }" + # The result should be a simple dict, the preproc returns a wrapped class, dict() will remove it + return dict(result) diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/librispeech.py b/tools/model_zoo/test_generator/sample_generator/dataset/librispeech.py new file mode 100644 index 00000000000..520e3edaebc --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/librispeech.py @@ -0,0 +1,47 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseDataset, ValidationDatasetHFIteratorMixin + + +class LibriSpeechASR(ValidationDatasetHFIteratorMixin, BaseDataset): + @property + def url(self): + return "librispeech_asr" + + @property + def split(self): + return "validation.clean" + + @staticmethod + def name(): + return "librispeech-asr" + + def transform(self, inputs, data, prepocess_fn): + result = prepocess_fn(data["audio"]["array"], + data["audio"]["sampling_rate"]) + inputs, keys = sorted(inputs), sorted(list(result.keys())) + assert inputs == keys, f"{inputs = } == {keys = }" + # The result should be a simple dict, the preproc returns a wrapped class, dict() will remove it + return dict(result) diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/prompts.py b/tools/model_zoo/test_generator/sample_generator/dataset/prompts.py new file mode 100644 index 00000000000..1ddb22c7e6f --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/prompts.py @@ -0,0 +1,121 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseDataset +from collections import namedtuple + +Prompt = namedtuple('Prompt', ['name', 'prompt', 'negative_prompt']) + +# source: https://huggingface.co/spaces/google/sdxl/blob/main/app.py#L19-L70 +style_list = [ + Prompt( + name="Cinematic", + prompt= + "cinematic still, emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy", + negative_prompt= + "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured", + ), + Prompt( + name="Photographic", + prompt= + "cinematic photo, 35mm photograph, film, bokeh, professional, 4k, highly detailed", + negative_prompt= + "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly", + ), + Prompt( + name="Anime", + prompt= + "anime artwork, anime style, key visual, vibrant, studio anime, highly detailed", + negative_prompt= + "photo, deformed, black and white, realism, disfigured, low contrast", + ), + Prompt( + name="Manga", + prompt= + "manga style, vibrant, high-energy, detailed, iconic, Japanese comic style", + negative_prompt= + "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style", + ), + Prompt( + name="Digital Art", + prompt= + "concept art, digital artwork, illustrative, painterly, matte painting, highly detailed", + negative_prompt="photo, photorealistic, realism, ugly", + ), + Prompt( + name="Pixel art", + prompt="pixel-art, low-res, blocky, pixel art style, 8-bit graphics", + negative_prompt= + "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic", + ), + Prompt( + name="Fantasy art", + prompt= + "ethereal fantasy concept art, magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy", + negative_prompt= + "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white", + ), + Prompt( + name="Neonpunk", + prompt= + "neonpunk style, cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional", + negative_prompt= + "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured", + ), + Prompt( + name="3D Model", + prompt= + "professional 3d model, octane render, highly detailed, volumetric, dramatic lighting", + negative_prompt="ugly, deformed, noisy, low poly, blurry, painting", + ), +] + + +class StylePrompts(BaseDataset): + @property + def url(self): + return "" + + @property + def split(self): + return "" + + @staticmethod + def name(): + return "style-prompts" + + def __iter__(self): + print(f"Load dataset for {self.name()}") + self.dataset = iter(style_list) + return self.dataset + + def __next__(self): + return next(self.dataset) + + def transform(self, inputs, data, prepocess_fn): + result = prepocess_fn(data.prompt, data.negative_prompt) + inputs, keys = sorted(inputs), sorted(list(result.keys())) + assert inputs == keys, f"{inputs = } == {keys = }" + # The result should be a simple dict, the preproc returns a wrapped class, dict() will remove it + return dict(result) diff --git a/tools/model_zoo/test_generator/sample_generator/dataset/squad.py b/tools/model_zoo/test_generator/sample_generator/dataset/squad.py new file mode 100644 index 00000000000..f47d45e0549 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/dataset/squad.py @@ -0,0 +1,83 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseDataset, ValidationDatasetHFIteratorMixin +from datasets import load_dataset + + +class SQuADTransformMixin(object): + def transform(self, inputs, data, prepocess_fn): + result = prepocess_fn(data["question"], + data["context"], + max_length=384) + inputs, keys = sorted(inputs), sorted(list(result.keys())) + assert inputs == keys, f"{inputs = } == {keys = }" + # The result should be a simple dict, the preproc returns a wrapped class, dict() will remove it + return dict(result) + + +class SQuADv1_1(SQuADTransformMixin, BaseDataset): + @property + def url(self): + return "https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/master/dataset/dev-v1.1.json" + + @property + def split(self): + return "validation" + + @staticmethod + def name(): + return "squad-v1.1" + + def __iter__(self): + print(f"Load dataset from {self.url}") + self.dataset = ({ + "context": paragraph["context"], + "question": qas["question"] + } for data in load_dataset("json", + data_files={"val": self.url}, + split="val", + field="data", + streaming=True) + for paragraph in data["paragraphs"] + for qas in paragraph["qas"]) + return self.dataset + + def __next__(self): + return next(self.dataset) + + +class SQuAD_HF(ValidationDatasetHFIteratorMixin, SQuADTransformMixin, + BaseDataset): + @property + def url(self): + return "squad" + + @property + def split(self): + return "validation" + + @staticmethod + def name(): + return "squad-hf" diff --git a/tools/model_zoo/test_generator/sample_generator/generator/__init__.py b/tools/model_zoo/test_generator/sample_generator/generator/__init__.py new file mode 100644 index 00000000000..51f7ed2506c --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/generator/__init__.py @@ -0,0 +1,28 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .generic import generate_test_dataset +from .diffusion import generate_diffusion_data + +__all__ = ["generate_test_dataset", "generate_diffusion_data"] diff --git a/tools/model_zoo/test_generator/sample_generator/generator/diffusion.py b/tools/model_zoo/test_generator/sample_generator/generator/diffusion.py new file mode 100644 index 00000000000..176a0167ddf --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/generator/diffusion.py @@ -0,0 +1,164 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import os +import onnxruntime as ort +import numpy as np +import torch + +from .generic import _inference +from ..utils import get_model_io + + +class DiffusionStage(object): + def __init__(self, model_path): + self.model_path = model_path + self._initilize() + + def _initilize(self): + self.inputs, self.outputs = get_model_io(self.model_path) + self.session = ort.InferenceSession(self.model_path) + self.folder_name_prefix = f"{os.path.dirname(self.model_path)}/test_data_set" + + def inference(self, input_data_map, test_idx): + folder_name = f"{self.folder_name_prefix}_{test_idx}" + output_data_map = _inference( + self.session, + input_data_map, + self.outputs, + folder_name, + ) + return output_data_map + + +def generate_diffusion_data(model, + image_dataset, + prompt_dataset, + output_folder_prefix=None, + sample_limit=None, + decode_limit=None): + assert model.is_diffuser() == True, "Only diffuser supported" + output_path = f"{output_folder_prefix or 'generated'}/diffusion/{model.name()}" + + try: + model_paths = model.get_model(output_path) + except Exception as e: + print(f"Something went wrong:\n{e}\nSkipping model...") + return + + is_xl = 'xl' in model.name() + assert len(model_paths) == (4 + is_xl) + text_encoder = DiffusionStage(model_paths[0]) + text_encoder_2 = DiffusionStage(model_paths[1]) if is_xl else None + vae_encoder = DiffusionStage(model_paths[1 + is_xl]) + unet = DiffusionStage(model_paths[2 + is_xl]) + vae_decoder = DiffusionStage(model_paths[3 + is_xl]) + + print('\n'.join([ + f"Creating {stage.folder_name_prefix}s..." for stage in + [text_encoder, text_encoder_2, vae_encoder, unet, vae_decoder] + if stage is not None + ])) + + test_idx = 0 + scale = 7.0 + seed = 42 + # TODO: get it from latent size + size = 128 if is_xl else 64 + sample_shape = (1, 4, size, size) + # TODO from config? + vae_scaling_factor = 0.18215 + noise = torch.randn(sample_shape, generator=torch.manual_seed(seed)) + time_ids = np.array([[size * 8, size * 8, 0, 0, size * 8, size * 8]] * 2, + dtype=np.float32) + + for idx, (image, prompt) in enumerate(zip(image_dataset, prompt_dataset)): + text_encoder_input_data_map = prompt_dataset.transform( + text_encoder.inputs, prompt, model.text_preprocess) + text_encoder_output_data_map = text_encoder.inference( + text_encoder_input_data_map, test_idx) + + if is_xl: + text_encoder_2_input_data_map = prompt_dataset.transform( + text_encoder_2.inputs, prompt, model.text_preprocess_2) + text_encoder_2_output_data_map = text_encoder_2.inference( + text_encoder_2_input_data_map, test_idx) + text_encoder_output_data_map['last_hidden_state'] = np.concatenate( + (text_encoder_output_data_map['last_hidden_state'], + text_encoder_2_output_data_map['last_hidden_state']), + axis=2) + + vae_encoder_input_data_map = image_dataset.transform( + vae_encoder.inputs, image, model.image_preprocess) + vae_encoder_output_data_map = vae_encoder.inference( + vae_encoder_input_data_map, test_idx) + + model.scheduler.set_timesteps(decode_limit) + latents = torch.from_numpy( + vae_encoder_output_data_map["latent_sample"]) * vae_scaling_factor + latents = model.scheduler.add_noise(latents, noise, + model.scheduler.timesteps[:1]) + + for step, t in enumerate(model.scheduler.timesteps): + latents_model_input = torch.concatenate([latents] * 2) + latents_model_input = model.scheduler.scale_model_input( + latents_model_input, t).numpy().astype(np.float32) + timestep = np.atleast_1d(t.numpy().astype(np.int64)) + + unet_input_data_map = { + 'sample': + latents_model_input, + 'encoder_hidden_states': + text_encoder_output_data_map['last_hidden_state'], + 'timestep': + timestep + } + if is_xl: + unet_input_data_map[ + 'text_embeds'] = text_encoder_2_output_data_map[ + 'text_embeds'] + unet_input_data_map['time_ids'] = time_ids + + unet_output_data_map = unet.inference( + unet_input_data_map, test_idx * decode_limit + step) + + noise_pred_text, noise_pred_uncond = np.array_split( + unet_output_data_map['out_sample'], 2) + + noise_pred = noise_pred_uncond + scale * (noise_pred_text - + noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = model.scheduler.step(torch.from_numpy(noise_pred), t, + latents).prev_sample + + latents = 1 / vae_scaling_factor * latents + vae_decoder_input_data_map = { + "latent_sample": latents.numpy().astype(np.float32) + } + _ = vae_decoder.inference(vae_decoder_input_data_map, test_idx) + + test_idx += 1 + if sample_limit and sample_limit - 1 <= idx: + break diff --git a/tools/model_zoo/test_generator/sample_generator/generator/generic.py b/tools/model_zoo/test_generator/sample_generator/generator/generic.py new file mode 100644 index 00000000000..cd83e877a85 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/generator/generic.py @@ -0,0 +1,83 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import os +import onnxruntime as ort + +from ..utils import get_model_io, numpy_to_pb + + +def _inference(session, input_data_map, outputs, folder_name): + input_pb_name = "input_{}.pb" + output_pb_name = "output_{}.pb" + os.makedirs(folder_name, exist_ok=True) + for input_idx, (input_name, + input_data) in enumerate(input_data_map.items()): + numpy_to_pb(input_name, input_data, + f"{folder_name}/{input_pb_name.format(input_idx)}") + + ort_result = session.run(outputs, input_data_map) + output_data_map = { + output_name: result_data + for (output_name, result_data) in zip(outputs, ort_result) + } + for output_idx, (output_name, + result_data) in enumerate(output_data_map.items()): + numpy_to_pb(output_name, result_data, + f"{folder_name}/{output_pb_name.format(output_idx)}") + + return output_data_map + + +def generate_test_dataset(model, + dataset, + output_folder_prefix=None, + sample_limit=None, + decode_limit=None): + assert model.is_diffuser() == False, "Only base and decoder supported" + output_path = f"{output_folder_prefix or 'generated'}/{dataset.name()}/{model.name()}" + folder_name_prefix = f"{output_path}/test_data_set" + + model_path = model.get_model(output_path) + inputs, outputs = get_model_io(model_path) + + sess = ort.InferenceSession(model_path) + print(f"Creating {folder_name_prefix}s...") + test_idx = 0 + for idx, data in enumerate(dataset): + input_data_map = dataset.transform(inputs, data, model.preprocess) + is_eos, decode_idx = False, 0 + while not is_eos and not (decode_limit and decode_limit <= decode_idx): + folder_name = f"{folder_name_prefix}_{test_idx}" + + output_data_map = _inference(sess, input_data_map, outputs, + folder_name) + + is_eos = not model.is_decoder() or model.decode_step( + input_data_map, output_data_map) + test_idx += 1 + decode_idx += 1 + + if sample_limit and sample_limit - 1 <= idx: + break diff --git a/tools/model_zoo/test_generator/sample_generator/model/__init__.py b/tools/model_zoo/test_generator/sample_generator/model/__init__.py new file mode 100644 index 00000000000..8284ffed998 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/__init__.py @@ -0,0 +1,25 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +__all__ = ["audio", "image", "text", "hybrid", "diffusion"] diff --git a/tools/model_zoo/test_generator/sample_generator/model/audio.py b/tools/model_zoo/test_generator/sample_generator/model/audio.py new file mode 100644 index 00000000000..7a4ab5ddec0 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/audio.py @@ -0,0 +1,90 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseModel, DecoderModel, OptimumHFModelDownloadMixin +import numpy as np +from transformers import AutoFeatureExtractor + + +class AutoFeatureExtractorHFMixin(object): + + _processor = None + + @property + def processor(self): + if self._processor is None: + self._processor = AutoFeatureExtractor.from_pretrained( + self.model_id) + return self._processor + + def preprocess(self, audio_data, sampling_rate): + return self.processor(audio_data, + sampling_rate=sampling_rate, + return_tensors="np") + + +class Wav2Vec2_base_960h(OptimumHFModelDownloadMixin, + AutoFeatureExtractorHFMixin, BaseModel): + @property + def model_id(self): + return "facebook/wav2vec2-base-960h" + + @staticmethod + def name(): + return "wav2vec2-base-960h" + + +class WhisperSmallEn(OptimumHFModelDownloadMixin, AutoFeatureExtractorHFMixin, + DecoderModel): + def __init__(self): + # Whisper specific part + # TODO get these from config + self.eos_token_id = 50256 # "<|endoftext|>" + decoder_start_token_id = 50257 # <|startoftranscript|> + notimestamps = 50362 # <|notimestamps|> + sot = [decoder_start_token_id, notimestamps] + max_length = 448 + self.initial_input_ids = np.array( + [sot + [self.eos_token_id] * (max_length - len(sot))]) + + @property + def model_id(self): + return "openai/whisper-small.en" + + @staticmethod + def name(): + return "whisper-small-en" + + def preprocess(self, *args, **kwargs): + # result only contains encoder data, extend it with decoder + result = super().preprocess(*args, **kwargs) + result["decoder_input_ids"] = np.copy(self.initial_input_ids) + return result + + def decode_step(self, input_map, output_map): + timestep = np.argmax( + input_map["decoder_input_ids"][0] == self.eos_token_id) + new_token = np.argmax(output_map["logits"][0][timestep - 1]) + input_map["decoder_input_ids"][0][timestep] = new_token + return new_token == self.eos_token_id diff --git a/tools/model_zoo/test_generator/sample_generator/model/base.py b/tools/model_zoo/test_generator/sample_generator/model/base.py new file mode 100644 index 00000000000..d45198c84b9 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/base.py @@ -0,0 +1,114 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import abc +import os +from optimum.exporters.onnx import main_export +from ..utils import download + + +class BaseModel(abc.ABC): + @property + @abc.abstractmethod + def model_id(self): + pass + + @staticmethod + @abc.abstractmethod + def name(): + pass + + @property + def task(self): + return "auto" + + @abc.abstractmethod + def get_model(self, folder, force_download): + pass + + @abc.abstractmethod + def preprocess(self, *args, **kwargs): + pass + + def is_decoder(self): + return False + + def is_diffuser(self): + return False + + +class DecoderModel(BaseModel): + def is_decoder(self): + return True + + @abc.abstractmethod + def decode_step(self, *args, **kwargs): + pass + + +class DiffusionModel(BaseModel): + def is_diffuser(self): + return True + + @abc.abstractmethod + def get_models(self, folder, models, force_download): + pass + + @abc.abstractmethod + def text_preprocess(self, *args, **kwargs): + pass + + @abc.abstractmethod + def text_preprocess_2(self, *args, **kwargs): + pass + + @abc.abstractmethod + def image_preprocess(self, *args, **kwargs): + pass + + @abc.abstractmethod + def scheduler(self, *args, **kwargs): + pass + + +class SingleModelDownloadMixin(object): + def get_model(self, output_folder, force_download=False): + filepath = f"{output_folder}/model.onnx" + if force_download or not os.path.isfile(filepath): + print(f"Download model from {self.model_id} to {filepath}") + download(self.model_id, filepath) + return filepath + + +class OptimumHFModelDownloadMixin(object): + def get_model(self, folder, force_download=False): + filepath = f"{folder}/model.onnx" + if force_download or not os.path.isfile(filepath): + print(f"Download model from {self.model_id} to {filepath}") + main_export( + self.model_id, + output=folder, + monolith=True, # forces export into one model + task=self.task) + return filepath diff --git a/tools/model_zoo/test_generator/sample_generator/model/diffusion.py b/tools/model_zoo/test_generator/sample_generator/model/diffusion.py new file mode 100644 index 00000000000..fd081064573 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/diffusion.py @@ -0,0 +1,188 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import DiffusionModel +from transformers import AutoTokenizer +from diffusers import EulerDiscreteScheduler +from diffusers.image_processor import VaeImageProcessor +from optimum.exporters.onnx import main_export +import numpy as np +import os +from PIL import ImageOps + + +class OptimumHFDiffusionModelDownloadMixin(object): + def get_models(self, folder, models, force_download=False): + filepaths = [f"{folder}/{model}/model.onnx" for model in models] + if force_download or not all( + os.path.isfile(filepath) for filepath in filepaths): + print(f"Download model from {self.model_id} to {filepaths}") + main_export( + self.model_id, + output=folder, + monolith=True, # forces export into one model + task=self.task) + return filepaths + + +class AutoTokenizerHFMixin(object): + + _tokenizer = None + _tokenizer_2 = None + + @property + def tokenizer(self): + if self._tokenizer is None: + self._tokenizer = AutoTokenizer.from_pretrained( + self.model_id, subfolder="tokenizer") + return self._tokenizer + + @property + def tokenizer_2(self): + if self._tokenizer_2 is None: + self._tokenizer_2 = AutoTokenizer.from_pretrained( + self.model_id, subfolder="tokenizer_2") + return self._tokenizer_2 + + def text_preprocess(self, prompt, max_length=77, version_2=False): + tokenizer = self.tokenizer_2 if version_2 else self.tokenizer + return tokenizer(prompt, + padding='max_length', + max_length=max_length, + return_tensors="np") + + +class VAEImageProcessorHFMixin(object): + + _processor = None + + @property + def processor(self): + if self._processor is None: + self._processor = VaeImageProcessor() + return self._processor + + def image_preprocess(self, image_data): + return self.processor.preprocess(image_data).numpy() + + +class EulerDiscreteSchedulerHFMixin(object): + + _scheduler = None + + @property + def scheduler(self): + if self._scheduler is None: + self._scheduler = EulerDiscreteScheduler.from_pretrained( + self.model_id, subfolder="scheduler") + return self._scheduler + + +class StableDiffusion21(OptimumHFDiffusionModelDownloadMixin, + AutoTokenizerHFMixin, VAEImageProcessorHFMixin, + EulerDiscreteSchedulerHFMixin, DiffusionModel): + def __init__(self): + # it should be in pipeline exec order + self.models = ('text_encoder', 'vae_encoder', 'unet', 'vae_decoder') + + @property + def model_id(self): + return "stabilityai/stable-diffusion-2-1" + + @staticmethod + def name(): + return "stable-diffusion-2-1" + + def get_model(self, folder, force_download=False): + return super().get_models(folder, self.models, force_download) + + def preprocess(self, *args, **kwargs): + raise RuntimeError("Call text_preprocess or image_preprocess directly") + + def text_preprocess(self, *args, **kwargs): + prompt, negative_prompt = args + prompt_result = super().text_preprocess(prompt, **kwargs) + negative_prompt_result = super().text_preprocess( + negative_prompt, **kwargs) + result = { + 'input_ids': + np.concatenate( + (prompt_result['input_ids'], + negative_prompt_result['input_ids'])).astype(np.int32) + } + return result + + def text_preprocess_2(self, *args, **kwargs): + raise RuntimeError("No tokenizer_2 for SD21 model") + + def image_preprocess(self, *args, **kwargs): + resized_image = ImageOps.fit(args[0], (512, 512)) + result = super().image_preprocess(resized_image, **kwargs) + return {"sample": result} + + +class StableDiffusionXL(OptimumHFDiffusionModelDownloadMixin, + AutoTokenizerHFMixin, VAEImageProcessorHFMixin, + EulerDiscreteSchedulerHFMixin, DiffusionModel): + def __init__(self): + # it should be in pipeline exec order + self.models = ('text_encoder', 'text_encoder_2', 'vae_encoder', 'unet', + 'vae_decoder') + + @property + def model_id(self): + return "stabilityai/stable-diffusion-xl-base-1.0" + + @staticmethod + def name(): + return "stable-diffusion-xl" + + def get_model(self, folder, force_download=False): + return super().get_models(folder, self.models, force_download) + + def preprocess(self, *args, **kwargs): + raise RuntimeError("Call text_preprocess or image_preprocess directly") + + def text_preprocess(self, *args, **kwargs): + prompt, negative_prompt = args + dtype = np.int64 if kwargs.get('version_2', False) else np.int32 + prompt_result = super().text_preprocess(prompt, **kwargs) + negative_prompt_result = super().text_preprocess( + negative_prompt, **kwargs) + result = { + 'input_ids': + np.concatenate((prompt_result['input_ids'], + negative_prompt_result['input_ids'])).astype(dtype) + } + return result + + def text_preprocess_2(self, *args, **kwargs): + new_kwargs = dict(kwargs) + new_kwargs['version_2'] = True + return self.text_preprocess(*args, **new_kwargs) + + def image_preprocess(self, *args, **kwargs): + resized_image = ImageOps.fit(args[0], (1024, 1024)) + result = super().image_preprocess(resized_image, **kwargs) + return {"sample": result} diff --git a/tools/model_zoo/test_generator/sample_generator/model/hybrid.py b/tools/model_zoo/test_generator/sample_generator/model/hybrid.py new file mode 100644 index 00000000000..9e387f9985e --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/hybrid.py @@ -0,0 +1,70 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseModel, OptimumHFModelDownloadMixin +from ..utils import get_imagenet_classes +from transformers import AutoProcessor + + +class AutoProcessorHFMixin(object): + + _processor = None + + @property + def processor(self): + if self._processor is None: + self._processor = AutoProcessor.from_pretrained(self.model_id) + return self._processor + + def preprocess(self, images, text, max_length=384): + return self.processor(images=images, + text=text, + max_length=max_length, + padding='max_length', + return_tensors="np") + + +class ClipVitLargePatch14(OptimumHFModelDownloadMixin, AutoProcessorHFMixin, + BaseModel): + def __init__(self): + import random + random.seed(42) + # cache a few labels, the full 1000 label is overkill + self.imagenet_labels = random.sample(get_imagenet_classes(), 10) + + @property + def model_id(self): + return "openai/clip-vit-large-patch14" + + @staticmethod + def name(): + return "clip-vit-large-patch14" + + def preprocess(self, *args, **kwargs): + # extend image with imagenet labels + new_args, new_kwargs = list(args), kwargs + new_args.append(self.imagenet_labels) + new_kwargs["max_length"] = 77 + result = super().preprocess(*new_args, **new_kwargs) + return result diff --git a/tools/model_zoo/test_generator/sample_generator/model/image.py b/tools/model_zoo/test_generator/sample_generator/model/image.py new file mode 100644 index 00000000000..f2a00d1e54a --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/image.py @@ -0,0 +1,114 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseModel, SingleModelDownloadMixin, OptimumHFModelDownloadMixin +from .preprocess import process_image + +from transformers import AutoImageProcessor +import timm + + +class AutoImageProcessorHFMixin(object): + + _processor = None + + @property + def processor(self): + if self._processor is None: + self._processor = AutoImageProcessor.from_pretrained(self.model_id) + return self._processor + + def preprocess(self, image_data): + return self.processor(image_data, return_tensors="np") + + +class ResNet50_v1(SingleModelDownloadMixin, BaseModel): + @property + def model_id(self): + return "https://zenodo.org/record/2592612/files/resnet50_v1.onnx" + + @staticmethod + def name(): + return "resnet50_v1" + + def preprocess(self, image_data): + IMAGENET_MEANS = [123.68, 116.78, 103.94] # RGB + return { + "input_tensor:0": process_image(image_data, means=IMAGENET_MEANS) + } + + +class ResNet50_v1_5(OptimumHFModelDownloadMixin, AutoImageProcessorHFMixin, + BaseModel): + @property + def model_id(self): + return "microsoft/resnet-50" + + @staticmethod + def name(): + return "resnet50_v1.5" + + +class VitBasePatch16_224(OptimumHFModelDownloadMixin, + AutoImageProcessorHFMixin, BaseModel): + @property + def model_id(self): + return "google/vit-base-patch16-224" + + @staticmethod + def name(): + return "vit-base-patch16-224" + + +class TIMM_MobileNetv3_large(OptimumHFModelDownloadMixin, BaseModel): + def __init__(self): + data_config = timm.data.resolve_model_data_config( + timm.create_model(self.model_id, pretrained=True)) + self.processor = timm.data.create_transform(**data_config, + is_training=False) + + @property + def model_id(self): + return "timm/mobilenetv3_large_100.ra_in1k" + + @staticmethod + def name(): + return "timm-mobilenetv3-large" + + def preprocess(self, image_data): + return { + "pixel_values": + self.processor(image_data).unsqueeze(0).cpu().detach().numpy() + } + + +# TODO enable it when BiT is supported by optimum +class Bit50(OptimumHFModelDownloadMixin, AutoImageProcessorHFMixin, BaseModel): + @property + def model_id(self): + return "google/bit-50" + + @staticmethod + def name(): + return "bit-50" diff --git a/tools/model_zoo/test_generator/sample_generator/model/preprocess.py b/tools/model_zoo/test_generator/sample_generator/model/preprocess.py new file mode 100644 index 00000000000..5bed88318dc --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/preprocess.py @@ -0,0 +1,73 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import cv2 +import numpy as np + + +def process_image(pil_image, shape=(224, 224), means=None): + # OpenCV default is BGR, the image is in RGB + image = cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR) + # But we need it to be in RGB, not sure how to import and not convert + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + w, h = shape + image = resize_with_aspectratio(image, h, w, inter_pol=cv2.INTER_AREA) + image = center_crop(image, h, w) + image = np.asarray(image, dtype="float32") + # Normalize image. + if means: + means = np.array(means, dtype=np.float32) + image -= means + # RGB is channel_last (HWC), transpose to channel_first (CHW) + image = image.transpose([2, 0, 1]) + image = image[np.newaxis, :] + return image + + +def center_crop(img, out_height, out_width): + height, width, _ = img.shape + left = int((width - out_width) / 2) + right = int((width + out_width) / 2) + top = int((height - out_height) / 2) + bottom = int((height + out_height) / 2) + img = img[top:bottom, left:right] + return img + + +def resize_with_aspectratio(img, + out_height, + out_width, + scale=87.5, + inter_pol=cv2.INTER_LINEAR): + height, width, _ = img.shape + new_height = int(100.0 * out_height / scale) + new_width = int(100.0 * out_width / scale) + if height > width: + w = new_width + h = int(new_height * height / width) + else: + h = new_height + w = int(new_width * width / height) + img = cv2.resize(img, (w, h), interpolation=inter_pol) + return img diff --git a/tools/model_zoo/test_generator/sample_generator/model/text.py b/tools/model_zoo/test_generator/sample_generator/model/text.py new file mode 100644 index 00000000000..a8bf294ac88 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/model/text.py @@ -0,0 +1,254 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .base import BaseModel, DecoderModel, OptimumHFModelDownloadMixin +import numpy as np +from transformers import AutoTokenizer + + +class AutoTokenizerHFMixin(object): + + _processor = None + + @property + def processor(self): + if self._processor is None: + self._processor = AutoTokenizer.from_pretrained(self.model_id) + return self._processor + + def preprocess(self, + question, + context, + max_length, + truncation='only_second'): + return self.processor(question, + context, + padding='max_length', + max_length=max_length, + truncation=truncation, + return_tensors="np") + + +class TextGenerationDecoderOnlyMixin(object): + def __init__(self): + # no pad token by default + self.processor.pad_token = self.processor.eos_token + + def preprocess(self, *args, **kwargs): + # swap squad's default "question - answer" order + new_args, new_kwargs = list(args), kwargs + new_args[0], new_args[1] = new_args[1], new_args[0] + new_kwargs["truncation"] = "only_first" + result = super().preprocess(*new_args, **new_kwargs) + + # result only contains "input_ids" and "attention_mask", extend it with "position_ids" + result["position_ids"] = np.arange(0, + len(result["input_ids"][0]), + dtype=np.int64) + result["position_ids"] = result["position_ids"][np.newaxis] + return result + + def decode_step(self, input_map, output_map): + timestep = np.argmax( + input_map["input_ids"][0] == self.processor.eos_token_id) + new_token = np.argmax(output_map["logits"][0][timestep - 1]) + input_map["input_ids"][0][timestep] = new_token + input_map["attention_mask"][0][timestep] = 1 + return new_token == self.processor.eos_token_id + + +class BERT_large_uncased(OptimumHFModelDownloadMixin, AutoTokenizerHFMixin, + BaseModel): + @property + def model_id(self): + return "google-bert/bert-large-uncased" + + @staticmethod + def name(): + return "bert-large-uncased" + + def preprocess(self, *args, **kwargs): + # use squad's question for masking + question, context = args + # replace first word with [MASK] + masked_question = question.replace(question[:question.index(' ')], + '[MASK]', 1) + # swap question-answer order + new_args = [context, masked_question] + result = super().preprocess(*new_args, **kwargs) + return result + + +class DistilBERT_base_cased_distilled_SQuAD(OptimumHFModelDownloadMixin, + AutoTokenizerHFMixin, BaseModel): + @property + def model_id(self): + return "distilbert/distilbert-base-cased-distilled-squad" + + @staticmethod + def name(): + return "distilbert-base-cased-distilled-squad" + + +class RobertaBaseSquad2(OptimumHFModelDownloadMixin, AutoTokenizerHFMixin, + BaseModel): + @property + def model_id(self): + return "deepset/roberta-base-squad2" + + @staticmethod + def name(): + return "roberta-base-squad2" + + +# Note: the inheritance order here important +class GPTJ(OptimumHFModelDownloadMixin, TextGenerationDecoderOnlyMixin, + AutoTokenizerHFMixin, DecoderModel): + @property + def model_id(self): + return "EleutherAI/gpt-j-6b" + + @property + def task(self): + # override to ignore "with-past" + return "text-generation" + + @staticmethod + def name(): + return "gpt-j" + + +# Note: the inheritance order here important +class Llama2_7b_chat_hf(OptimumHFModelDownloadMixin, + TextGenerationDecoderOnlyMixin, AutoTokenizerHFMixin, + DecoderModel): + @property + def model_id(self): + return "meta-llama/Llama-2-7b-chat-hf" + + @property + def task(self): + # override to ignore "with-past" + return "text-generation" + + @staticmethod + def name(): + return "llama2-7b-chat-hf" + + +# Note: the inheritance order here important +class Llama3_8b_instruct(OptimumHFModelDownloadMixin, + TextGenerationDecoderOnlyMixin, AutoTokenizerHFMixin, + DecoderModel): + @property + def model_id(self): + return "meta-llama/Meta-Llama-3-8B-Instruct" + + @property + def task(self): + # override to ignore "with-past" + return "text-generation" + + @staticmethod + def name(): + return "llama3-8b-instruct" + + +class T5_base(OptimumHFModelDownloadMixin, AutoTokenizerHFMixin, DecoderModel): + def __init__(self): + max_length = 384 # default for squad + # Note: no real start token, it requires the pad token + self.initial_input_ids = np.array( + [[self.processor.pad_token_id] + [self.processor.eos_token_id] * + (max_length - 1)]) + + @property + def model_id(self): + return "google-t5/t5-base" + + @property + def task(self): + # override to ignore "with-past" + return "text2text-generation" + + @staticmethod + def name(): + return "t5-base" + + def preprocess(self, *args, **kwargs): + # only use squad's question + new_args = ["translate English to French:", args[0]] + result = super().preprocess(*new_args, **kwargs) + result["decoder_input_ids"] = np.copy(self.initial_input_ids) + return result + + def decode_step(self, input_map, output_map): + timestep = np.argmax( + input_map["decoder_input_ids"][0] == self.processor.eos_token_id) + new_token = np.argmax(output_map["logits"][0][timestep - 1]) + input_map["decoder_input_ids"][0][timestep] = new_token + input_map["attention_mask"][0][timestep] = 1 + return new_token == self.processor.eos_token_id + + +class Gemma_2b_it(OptimumHFModelDownloadMixin, AutoTokenizerHFMixin, + DecoderModel): + @property + def model_id(self): + return "google/gemma-2b-it" + + @property + def task(self): + # override to ignore "with-past" + return "text-generation" + + @staticmethod + def name(): + return "gemma-2b-it" + + def preprocess(self, *args, **kwargs): + # swap squad's default "question - answer" order + new_args, new_kwargs = list(args), kwargs + new_args[0], new_args[1] = new_args[1], new_args[0] + new_kwargs["truncation"] = "only_first" + result = super().preprocess(*new_args, **new_kwargs) + # Note: padding-side is left, pos ids will be 1,1,1,...,0,1,2,3,... + result["position_ids"] = np.cumsum(result["attention_mask"][0], -1) - 1 + result["position_ids"][:np.argmax(result["position_ids"] == 0)] = 1 + result["position_ids"] = result["position_ids"][np.newaxis] + return result + + def decode_step(self, input_map, output_map): + # The result is in the last logits + new_token = np.argmax(output_map["logits"][0][-1]) + # Move everything left 1 step + input_map["input_ids"][0] = np.roll(input_map["input_ids"][0], -1) + input_map["input_ids"][0][-1] = new_token + input_map["attention_mask"][0] = np.roll( + input_map["attention_mask"][0], -1) + input_map["attention_mask"][0][-1] = 1 + input_map["position_ids"][0] = np.roll(input_map["position_ids"][0], + -1) + input_map["position_ids"][0][-1] += input_map["position_ids"][0][-2] + return new_token == self.processor.eos_token_id diff --git a/tools/model_zoo/test_generator/sample_generator/utils/__init__.py b/tools/model_zoo/test_generator/sample_generator/utils/__init__.py new file mode 100644 index 00000000000..cd41a4e18ce --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/utils/__init__.py @@ -0,0 +1,26 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +from .utils import get_model_io, download, numpy_to_pb, get_imagenet_classes +__all__ = ["get_model_io", "download", "numpy_to_pb", "get_imagenet_classes"] diff --git a/tools/model_zoo/test_generator/sample_generator/utils/utils.py b/tools/model_zoo/test_generator/sample_generator/utils/utils.py new file mode 100644 index 00000000000..89b630d68f6 --- /dev/null +++ b/tools/model_zoo/test_generator/sample_generator/utils/utils.py @@ -0,0 +1,78 @@ +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### +import onnx +import logging +import os +import requests + + +def get_model_io(model_path): + model = onnx.load(model_path) + outputs = [node.name for node in model.graph.output] + + input_all = [node.name for node in model.graph.input] + input_initializer = [node.name for node in model.graph.initializer] + inputs = list(set(input_all) - set(input_initializer)) + + return inputs, outputs + + +def numpy_to_pb(name, np_data, out_filename): + """Convert numpy data to a protobuf file.""" + + tensor = onnx.numpy_helper.from_array(np_data, name) + onnx.save_tensor(tensor, out_filename) + + +def download(url, filename, quiet=False): + try: + from tqdm import tqdm + except: + quiet = True + + logging.debug(f"Download {filename} from {url}") + response = requests.get(url, stream=True) + + total_size_in_bytes = int(response.headers.get("content-length", 0)) + block_size = 1024 + if not quiet: + progress_bar = tqdm(total=total_size_in_bytes, + unit="iB", + unit_scale=True) + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "wb") as f: + for data in response.iter_content(block_size): + if not quiet: + progress_bar.update(len(data)) + f.write(data) + if not quiet: + progress_bar.close() + + +def get_imagenet_classes(): + url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + response = requests.get(url) + return response.text.split("\n") diff --git a/tools/model_zoo/test_generator/test_models.sh b/tools/model_zoo/test_generator/test_models.sh new file mode 100755 index 00000000000..9b28b7f6d26 --- /dev/null +++ b/tools/model_zoo/test_generator/test_models.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +##################################################################################### +# The MIT License (MIT) +# +# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##################################################################################### + +set -e + +WORK_DIR="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_PATH=$(dirname $(dirname $(dirname $(readlink -f "$0"))))/test_runner.py +TESTER_SCRIPT="${TESTER:-$SCRIPT_PATH}" +ATOL="${ATOL:-0.001}" +RTOL="${RTOL:-0.001}" +TARGET="${TARGET:-gpu}" + +if [[ "${DEBUG:-0}" -eq 1 ]]; then + PIPE=/dev/stdout +else + PIPE=/dev/null +fi + +if [[ "${VERBOSE:-0}" -eq 1 ]]; then + set -x +fi + +# Iterate through input recursively, process any onnx file +function iterate() { + local dir="$1" + + for file in "$dir"/*; do + if [ -f "$file" ]; then + if [[ $file = *.onnx ]]; then + process "$file" + fi + fi + + if [ -d "$file" ]; then + iterate "$file" + fi + done +} + +# Test it with test_runner.py, both fp32 and fp16 +function process() { + local file="$1" + echo "INFO: process $file started" + test $file fp32 + test $file fp16 + echo "INFO: process $file finished" +} + +# Run test_runner.py and log if something goes wrong +function test() { + local file="$1" + echo "INFO: test $file ($2)" + model_folder="$(dirname $file)" + model_name="$(basename $model_folder)" + # avoid name conflict for non-unique submodels + if [[ "$model_folder" =~ "diffusion" ]]; then + model_name="$(basename $(dirname $model_folder))_${model_name}" + fi + flag="--atol $ATOL --rtol $RTOL --target $TARGET" + if [[ "$2" = "fp16" ]]; then + flag="$flag --fp16" + fi + EXIT_CODE=0 + python3 $TESTER_SCRIPT ${flag} ${model_folder}/ &> "$WORK_DIR/logs/$2/${model_name//\//_}.log" || EXIT_CODE=$? + if [[ "${EXIT_CODE:-0}" -ne 0 ]]; then + echo "WARNING: ${file} failed ($2)" + fi +} + +mkdir -p $WORK_DIR/logs/fp32/ $WORK_DIR/logs/fp16/ + +for arg in "$@"; do + iterate "$(dirname $(readlink -e $arg))/$(basename $arg)" +done