diff --git a/.github/workflows/ci_linux_x64-libshortfin.yml b/.github/workflows/ci_linux_x64-libshortfin.yml index babcf0245..793058570 100644 --- a/.github/workflows/ci_linux_x64-libshortfin.yml +++ b/.github/workflows/ci_linux_x64-libshortfin.yml @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -46,6 +46,7 @@ jobs: repository: iree-org/iree path: ${{ env.IREE_REPO_DIR }} submodules: false + ref: candidate-20240904.1006 - name: Initalize IREE submodules run : | @@ -83,8 +84,8 @@ jobs: - name: Install Python packages # TODO: Switch to `pip install -r requirements.txt -e libshortfin/`. run: | - pip install nanobind pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-tests.txt + pip freeze - name: Build libshortfin (full) run: | @@ -94,6 +95,7 @@ jobs: -DCMAKE_C_COMPILER=clang-18 \ -DCMAKE_CXX_COMPILER=clang++-18 \ -DCMAKE_LINKER_TYPE=LLD \ + -DSHORTFIN_BUNDLE_DEPS=ON \ -DCMAKE_PREFIX_PATH=${{ env.IREE_REPO_DIR }}/build/lib/cmake/IREE \ -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \ .. @@ -111,6 +113,8 @@ jobs: run: | mkdir ${{ env.LIBSHORTFIN_DIR }}/build-host-only cd ${{ env.LIBSHORTFIN_DIR }}/build-host-only + # In this configuration, also build static+dynamic in order to verify + # that path structurally works. cmake -GNinja \ -DCMAKE_C_COMPILER=clang-18 \ -DCMAKE_CXX_COMPILER=clang++-18 \ @@ -118,5 +122,7 @@ jobs: -DCMAKE_PREFIX_PATH=${{ env.IREE_REPO_DIR }}/build/lib/cmake/IREE \ -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \ -DSHORTFIN_HAVE_AMDGPU=OFF \ + -DSHORTFIN_BUILD_STATIC=ON \ + -DSHORTFIN_BUILD_DYNAMIC=ON \ .. cmake --build . --target all diff --git a/.github/workflows/ci_linux_x64_asan-libshortfin.yml b/.github/workflows/ci_linux_x64_asan-libshortfin.yml index 14aa26bda..c998fbc77 100644 --- a/.github/workflows/ci_linux_x64_asan-libshortfin.yml +++ b/.github/workflows/ci_linux_x64_asan-libshortfin.yml @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -23,16 +23,19 @@ env: PYENV_ROOT: ${{ github.workspace }}/pyenv PYENV_REF: 9ecd803bffaffb949fbdd8c70cb086227f6a3202 # v2.4.10 PYTHON_VER: 3.12.3 - CACHE_ASAN_VER: 1 + CACHE_ASAN_VER: 2 CACHE_DEPS_VER: 1 IREE_SOURCE_DIR: ${{ github.workspace }}/iree LIBSHORTFIN_DIR: ${{ github.workspace }}/libshortfin/ - jobs: setup-python-asan: name: Setup Python ASan runs-on: ubuntu-24.04 + env: + # The Python build process leaks. Here we just disable leak checking vs + # being more precise. + ASAN_OPTIONS: detect_leaks=0 steps: - name: Cache Python ASan @@ -64,15 +67,18 @@ jobs: cd ${{ env.PYENV_ROOT }} src/configure && make -C src export PATH=${{ env.PYENV_ROOT }}/bin:$PATH && eval "$(pyenv init -)" - CC=clang-18 CXX=clang++-18 LDFLAGS="-lstdc++" PYTHON_CONFIGURE_OPTS="--with-address-sanitizer" pyenv install -v -g ${{ env.PYTHON_VER }} - pyenv global ${{ env.PYTHON_VER }}-debug + CC=clang-18 CXX=clang++-18 LDFLAGS="-lstdc++" PYTHON_CONFIGURE_OPTS="--with-address-sanitizer" pyenv install -v ${{ env.PYTHON_VER }} + pyenv global ${{ env.PYTHON_VER }} build-and-test: name: Build and test libshortfin needs: [setup-python-asan] runs-on: ubuntu-24.04 - + env: + # TODO(#151): Don't ignore ODR violations + ASAN_OPTIONS: detect_odr_violation=0 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/libshortfin/build_tools/python_lsan_suppressions.txt steps: - name: Install dependencies run: | @@ -90,6 +96,7 @@ jobs: repository: iree-org/iree path: ${{ env.IREE_SOURCE_DIR }} submodules: false + ref: candidate-20240904.1006 - name: Initalize IREE submodules run : | @@ -124,6 +131,8 @@ jobs: run: | eval "$(pyenv init -)" pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-tests.txt + pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-iree-compiler.txt + pip freeze - name: Save Python dependencies cache if: steps.cache-python-deps-restore.outputs.cache-hit != 'true' @@ -134,9 +143,6 @@ jobs: key: ${{ steps.cache-python-deps-restore.outputs.cache-primary-key }} - name: Build libshortfin - env: - # TODO(#151): Don't ignore ODR violations - ASAN_OPTIONS=detect_odr_violation: 0 run: | eval "$(pyenv init -)" mkdir ${{ env.LIBSHORTFIN_DIR }}/build @@ -167,4 +173,4 @@ jobs: run: | eval "$(pyenv init -)" cd ${{ env.LIBSHORTFIN_DIR }} - pytest -m "not requires_amd_gpu" + pytest -m "not requires_amd_gpu" -s diff --git a/build_tools/integration_tests/llama_export_compile_serve.sh b/build_tools/integration_tests/llama_export_compile_serve.sh index 9409e6960..edd54b688 100755 --- a/build_tools/integration_tests/llama_export_compile_serve.sh +++ b/build_tools/integration_tests/llama_export_compile_serve.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/CMakeLists.txt b/libshortfin/CMakeLists.txt index c5a2f0f6a..86d0267a5 100644 --- a/libshortfin/CMakeLists.txt +++ b/libshortfin/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: @@ -18,6 +18,8 @@ project( VERSION 0.9 LANGUAGES C CXX) +include(CMakeDependentOption) + set(SOVERSION 1) set(CMAKE_C_STANDARD 11) @@ -34,10 +36,26 @@ endif() # build options option(SHORTFIN_BUILD_PYTHON_BINDINGS "Builds Python Bindings" OFF) option(SHORTFIN_BUILD_TESTS "Builds C++ tests" ON) -option(SHORTFIN_BUNDLE_DEPS "Download dependencies instead of using system libraries" OFF) +option(SHORTFIN_BUNDLE_DEPS "Download dependencies instead of using system libraries" ON) set(SHORTFIN_IREE_SOURCE_DIR "" CACHE FILEPATH "Path to IREE source") +# Options for building static or dynamic libraries. +option(SHORTFIN_BUILD_STATIC "Builds static libraries" OFF) +option(SHORTFIN_BUILD_DYNAMIC "Builds dynamic libraries" ON) +cmake_dependent_option(SHORTFIN_LINK_DYNAMIC "Links internal binaries against static libshortfin.a" ON "SHORTFIN_BUILD_DYNAMIC" OFF) +if(NOT SHORTFIN_BUILD_STATIC AND NOT SHORTFIN_BUILD_DYNAMIC) + message(FATAL_ERROR "One of SHORTFIN_BUILD_STATIC or SHORTFIN_BUILD_DYNAMIC must be ON") +endif() +message(STATUS "Shortfin build static = ${SHORTFIN_BUILD_STATIC}, dynamic = ${SHORTFIN_BUILD_DYNAMIC}") +if(SHORTFIN_LINK_DYNAMIC) + message(STATUS "Dynamic linking to shortfin") + set(SHORTFIN_LINK_LIBRARY_NAME "shortfin") +else() + message(STATUS "Static linking to shortfin-static") + set(SHORTFIN_LINK_LIBRARY_NAME "shortfin-static") +endif() + # Enabling ASAN. Note that this will work best if building in a completely # bundled fashion and with an ASAN rigged CPython. Otherwise, various LD_PRELOAD # hacks are needed. This is merely a develope convenience: people are more @@ -112,7 +130,7 @@ if (NOT SHORTFIN_IREE_SOURCE_DIR AND SHORTFIN_BUNDLE_DEPS) FetchContent_Declare( iree GIT_REPOSITORY https://github.com/iree-org/iree.git - GIT_TAG candidate-20240821.992 + GIT_TAG candidate-20240904.1006 # TODO: We shouldn't have to pull googletest when we are not building tests. # This needs to be fixed with IREE. GIT_SUBMODULES "third_party/benchmark third_party/cpuinfo third_party/flatcc third_party/hip-build-deps third_party/googletest" diff --git a/libshortfin/bindings/python/CMakeLists.txt b/libshortfin/bindings/python/CMakeLists.txt index 66e6476a6..f1e999783 100644 --- a/libshortfin/bindings/python/CMakeLists.txt +++ b/libshortfin/bindings/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: @@ -10,12 +10,13 @@ # build. - _shortfin_tracing.lib: Native library with tracing enabled (TODO). - # Others. -# Detect the installed nanobind package and import it into CMake -execute_process( - COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir - OUTPUT_STRIP_TRAILING_WHITESPACE - OUTPUT_VARIABLE nanobind_ROOT) -find_package(nanobind CONFIG REQUIRED) +# nanobind +FetchContent_Declare( + nanobind + GIT_REPOSITORY https://github.com/wjakob/nanobind.git + GIT_TAG 9641bb7151f04120013b812789b3ebdfa7e7324f # 2.1.0 +) +FetchContent_MakeAvailable(nanobind) nanobind_add_module(shortfin_python_extension NB_STATIC LTO array_binding.cc @@ -26,9 +27,7 @@ set_target_properties(shortfin_python_extension PROPERTIES OUTPUT_NAME "_shortfin_default/lib") target_link_libraries(shortfin_python_extension - # TODO: This should be configurable as to whether we link to the static - # or dynamic version. - PRIVATE shortfin + PRIVATE ${SHORTFIN_LINK_LIBRARY_NAME} ) nanobind_add_stub( diff --git a/libshortfin/bindings/python/_shortfin/__init__.py b/libshortfin/bindings/python/_shortfin/__init__.py index e47fd8fc6..ac43edaea 100644 --- a/libshortfin/bindings/python/_shortfin/__init__.py +++ b/libshortfin/bindings/python/_shortfin/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/_shortfin/asyncio_bridge.py b/libshortfin/bindings/python/_shortfin/asyncio_bridge.py index 63ded30e9..78aff49a6 100644 --- a/libshortfin/bindings/python/_shortfin/asyncio_bridge.py +++ b/libshortfin/bindings/python/_shortfin/asyncio_bridge.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/array_binding.cc b/libshortfin/bindings/python/array_binding.cc index 9858c2350..608aa4944 100644 --- a/libshortfin/bindings/python/array_binding.cc +++ b/libshortfin/bindings/python/array_binding.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -13,6 +13,23 @@ using namespace shortfin::array; namespace shortfin::python { namespace { +static const char DOCSTRING_ARRAY_COPY_FROM[] = + R"(Copy contents from a source array to this array. + +Equivalent to `dest_array.storage.copy_from(source_array.storage)`. +)"; + +static const char DOCSTRING_ARRAY_COPY_TO[] = + R"(Copy contents this array to a destination array. + +Equivalent to `dest_array.storage.copy_from(source_array.storage)`. +)"; + +static const char DOCSTRING_ARRAY_FILL[] = R"(Fill an array with a value. + +Equivalent to `array.storage.fill(pattern)`. +)"; + static const char DOCSTRING_STORAGE_DATA[] = R"(Access raw binary contents. Accessing `foo = storage.data` is equivalent to `storage.data.map(read=True)`. @@ -28,6 +45,23 @@ As with `map`, this will only work on buffers that are host visible, which includes all host buffers and device buffers created with the necessary access. )"; +static const char DOCSTRING_STORAGE_COPY_FROM[] = + R"(Copy contents from a source storage to this array. + +This operation executes asynchronously and the effect will only be visible +once the execution scope has been synced to the point of mutation. +)"; + +static const char DOCSTRING_STORAGE_FILL[] = R"(Fill a storage with a value. + +Takes as argument any value that can be interpreted as a buffer with the Python +buffer protocol of size 1, 2, or 4 bytes. The storage will be filled uniformly +with the pattern. + +This operation executes asynchronously and the effect will only be visible +once the execution scope has been synced to the point of mutation. +)"; + static const char DOCSTRING_STORAGE_MAP[] = R"(Create a mapping of the buffer contents in host memory. @@ -72,58 +106,47 @@ void BindArray(py::module_ &m) { .def(py::self == py::self) .def("__repr__", &DType::name); - m.attr("opaque8") = DType::opaque8(); - m.attr("opaque16") = DType::opaque16(); - m.attr("opaque32") = DType::opaque32(); - m.attr("opaque64") = DType::opaque64(); - m.attr("bool8") = DType::bool8(); - m.attr("int4") = DType::int4(); - m.attr("sint4") = DType::sint4(); - m.attr("uint4") = DType::uint4(); - m.attr("int8") = DType::int8(); - m.attr("sint8") = DType::sint8(); - m.attr("uint8") = DType::uint8(); - m.attr("int16") = DType::int16(); - m.attr("sint16") = DType::sint16(); - m.attr("uint16") = DType::uint16(); - m.attr("int32") = DType::int32(); - m.attr("sint32") = DType::sint32(); - m.attr("uint32") = DType::uint32(); - m.attr("int64") = DType::int64(); - m.attr("sint64") = DType::sint64(); - m.attr("uint64") = DType::uint64(); - m.attr("float16") = DType::float16(); - m.attr("float32") = DType::float32(); - m.attr("float64") = DType::float64(); - m.attr("bfloat16") = DType::bfloat16(); - m.attr("complex64") = DType::complex64(); - m.attr("complex128") = DType::complex128(); +#define SHORTFIN_DTYPE_HANDLE(et, ident) m.attr(#ident) = DType::ident(); +#include "shortfin/array/dtypes.inl" +#undef SHORTFIN_DTYPE_HANDLE // storage py::class_(m, "storage") + .def("__sfinv_marshal__", + [](device_array *self, py::capsule inv_capsule, int barrier) { + auto *inv = + static_cast(inv_capsule.data()); + static_cast(self) + ->AddAsInvocationArgument( + inv, static_cast(barrier)); + }) .def_static( "allocate_host", [](local::ScopedDevice &device, iree_device_size_t allocation_size) { - return storage::AllocateHost(device, allocation_size); + return storage::allocate_host(device, allocation_size); }, py::arg("device"), py::arg("allocation_size"), py::keep_alive<0, 1>()) .def_static( "allocate_device", [](local::ScopedDevice &device, iree_device_size_t allocation_size) { - return storage::AllocateDevice(device, allocation_size); + return storage::allocate_device(device, allocation_size); }, py::arg("device"), py::arg("allocation_size"), py::keep_alive<0, 1>()) - .def("fill", - [](storage &self, py::handle buffer) { - Py_buffer py_view; - int flags = PyBUF_FORMAT | PyBUF_ND; // C-Contiguous ND. - if (PyObject_GetBuffer(buffer.ptr(), &py_view, flags) != 0) { - throw py::python_error(); - } - PyBufferReleaser py_view_releaser(py_view); - self.Fill(py_view.buf, py_view.len); - }) - .def("copy_from", [](storage &self, storage &src) { self.CopyFrom(src); }) + .def( + "fill", + [](storage &self, py::handle buffer) { + Py_buffer py_view; + int flags = PyBUF_FORMAT | PyBUF_ND; // C-Contiguous ND. + if (PyObject_GetBuffer(buffer.ptr(), &py_view, flags) != 0) { + throw py::python_error(); + } + PyBufferReleaser py_view_releaser(py_view); + self.fill(py_view.buf, py_view.len); + }, + py::arg("pattern"), DOCSTRING_STORAGE_FILL) + .def( + "copy_from", [](storage &self, storage &src) { self.copy_from(src); }, + py::arg("source_storage"), DOCSTRING_STORAGE_COPY_FROM) .def( "map", [](storage &self, bool read, bool write, bool discard) { @@ -137,7 +160,7 @@ void BindArray(py::module_ &m) { } mapping *cpp_mapping = nullptr; py::object py_mapping = CreateMappingObject(&cpp_mapping); - self.MapExplicit( + self.map_explicit( *cpp_mapping, static_cast(access)); return py_mapping; @@ -154,12 +177,12 @@ void BindArray(py::module_ &m) { [](storage &self) { mapping *cpp_mapping = nullptr; py::object py_mapping = CreateMappingObject(&cpp_mapping); - *cpp_mapping = self.MapRead(); + *cpp_mapping = self.map_read(); return py_mapping; }, [](storage &self, py::handle buffer_obj) { PyBufferRequest src_info(buffer_obj, PyBUF_SIMPLE); - auto dest_data = self.MapWriteDiscard(); + auto dest_data = self.map_write_discard(); if (src_info.view().len > dest_data.size()) { throw std::invalid_argument( fmt::format("Cannot write {} bytes into buffer of {} bytes", @@ -169,6 +192,7 @@ void BindArray(py::module_ &m) { src_info.view().len); }, DOCSTRING_STORAGE_DATA) + .def(py::self == py::self) .def("__repr__", &storage::to_s); // mapping @@ -186,7 +210,7 @@ void BindArray(py::module_ &m) { int operator()(mapping &self, Py_buffer *view, int flags) { view->buf = self.data(); view->len = self.size(); - view->readonly = self.writable(); + view->readonly = !self.writable(); view->itemsize = 1; view->format = (char *)"B"; // Byte view->ndim = 1; @@ -219,6 +243,14 @@ void BindArray(py::module_ &m) { py_type, /*keep_alive=*/device.scope(), device_array::for_device(device, shape, dtype)); }) + .def("__sfinv_marshal__", + [](device_array *self, py::capsule inv_capsule, int barrier) { + auto *inv = + static_cast(inv_capsule.data()); + static_cast(self) + ->AddAsInvocationArgument( + inv, static_cast(barrier)); + }) .def_static("for_device", [](local::ScopedDevice &device, std::span shape, DType dtype) { @@ -243,7 +275,23 @@ void BindArray(py::module_ &m) { py::rv_policy::reference_internal) .def_prop_ro("storage", &device_array::storage, py::rv_policy::reference_internal) - .def("__repr__", &device_array::to_s); + + .def( + "fill", + [](py::handle_t self, py::handle buffer) { + self.attr("storage").attr("fill")(buffer); + }, + py::arg("pattern"), DOCSTRING_ARRAY_FILL) + .def("copy_from", &device_array::copy_from, py::arg("source_array"), + DOCSTRING_ARRAY_COPY_FROM) + .def("copy_to", &device_array::copy_to, py::arg("dest_array"), + DOCSTRING_ARRAY_COPY_TO) + .def("__repr__", &device_array::to_s) + .def("__str__", [](device_array &self) -> std::string { + auto contents = self.contents_to_s(); + if (!contents) return "<>"; + return *contents; + }); } } // namespace shortfin::python diff --git a/libshortfin/bindings/python/lib_ext.cc b/libshortfin/bindings/python/lib_ext.cc index 6072caa04..cc7c21190 100644 --- a/libshortfin/bindings/python/lib_ext.cc +++ b/libshortfin/bindings/python/lib_ext.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,6 +7,8 @@ #include "./lib_ext.h" #include "./utils.h" +#include "shortfin/array/array.h" +#include "shortfin/array/storage.h" #include "shortfin/local/async.h" #include "shortfin/local/messaging.h" #include "shortfin/local/process.h" @@ -24,6 +26,13 @@ namespace shortfin::python { namespace { +static const char DOCSTRING_PROGRAM_FUNCTION_INVOCATION[] = + R"(Creates an invocation object targeting the function. + +This is a low-level interface for performing an invocation, and it should be +used when precise, non-default control is needed. +)"; + class Refs { public: py::object asyncio_create_task = @@ -182,6 +191,59 @@ class PyProcess : public local::detail::BaseProcess { std::shared_ptr refs_; }; +void PyAddProgramInvocationArg(py::capsule &inv_capsule, py::handle arg) { + // See if the object implements our marshaling protocol. If it does, then + // We invoke the marshaling method with the Invocation wrapped as a capsule + // and the ProgramResourceBarrier. + py::object marshaler = py::getattr(arg, "__sfinv_marshal__", py::none()); + if (!marshaler.is_none()) { + marshaler(inv_capsule, + static_cast(local::ProgramResourceBarrier::DEFAULT)); + return; + } + + throw std::invalid_argument( + fmt::format("Unsupported argument type {} in call to ProgramFunction", + py::cast(py::repr(arg.type())))); +} + +local::ProgramInvocation::Future PyFunctionCall(local::ProgramFunction &self, + py::args args) { + auto inv = self.CreateInvocation(); + py::capsule inv_capsule(inv.get()); + for (py::handle arg : args) { + PyAddProgramInvocationArg(inv_capsule, arg); + } + return local::ProgramInvocation::Invoke(std::move(inv)); +} + +py::object PyRehydrateRef(local::ProgramInvocation *inv, + iree::vm_opaque_ref ref) { + auto type = ref.get()->type; + // Note that these accessors are dangerous as they assert/abort if + // process-wide registration is not done properly. We assume here that + // since we got a ref out that the basics are set up soundly, but if actually + // doing this on user/dynamic types, we would want to be more defensive. + // TODO: Don't just do a linear scan if we have more than a couple. + // TODO: Find a reliable way to statically cache the type id. + if (local::ProgramInvocationMarshalableFactory::invocation_marshalable_type< + array::device_array>() == type) { + // device_array + return py::cast(local::ProgramInvocationMarshalableFactory:: + CreateFromInvocationResultRef( + inv, std::move(ref))); + } else if (local::ProgramInvocationMarshalableFactory:: + invocation_marshalable_type() == type) { + // storage + return py::cast( + local::ProgramInvocationMarshalableFactory:: + CreateFromInvocationResultRef(inv, std::move(ref))); + } + throw std::invalid_argument( + fmt::format("Cannot marshal ref type {} to Python", + to_string_view(iree_vm_ref_type_name(type)))); +} + py::object RunInForeground(std::shared_ptr refs, local::System &self, py::object coro) { bool is_main_thread = @@ -189,9 +251,13 @@ py::object RunInForeground(std::shared_ptr refs, local::System &self, local::Worker &worker = self.init_worker(); py::object result = py::none(); + py::object py_exception = py::none(); auto done_callback = [&](py::handle future) { worker.Kill(); - result = future.attr("result")(); + py_exception = future.attr("exception")(); + if (py_exception.is_none()) { + result = future.attr("result")(); + } }; worker.CallThreadsafe([&]() { // Run within the worker we are about to donate to. @@ -231,13 +297,48 @@ py::object RunInForeground(std::shared_ptr refs, local::System &self, } self.Shutdown(); + + if (!py_exception.is_none()) { + // We got this exception from a future/user code, which could have done + // something nefarious. So type check it. + if (PyObject_IsInstance(py_exception.ptr(), PyExc_Exception)) { + PyErr_SetObject(py_exception.type().ptr(), py_exception.ptr()); + } else { + PyErr_SetObject(PyExc_RuntimeError, py_exception.ptr()); + } + throw py::python_error(); + } return result; } } // namespace NB_MODULE(lib, m) { - m.def("initialize", shortfin::GlobalInitialize); + py::register_exception_translator( + [](const std::exception_ptr &p, void * /*unused*/) { + try { + std::rethrow_exception(p); + } catch (shortfin::iree::error &e) { + PyObject *exc_type; + switch (e.code()) { + case IREE_STATUS_INVALID_ARGUMENT: + case IREE_STATUS_OUT_OF_RANGE: + exc_type = PyExc_ValueError; + break; + case IREE_STATUS_FAILED_PRECONDITION: + exc_type = PyExc_AssertionError; + break; + case IREE_STATUS_UNIMPLEMENTED: + exc_type = PyExc_NotImplementedError; + break; + default: + exc_type = PyExc_RuntimeError; + } + PyErr_SetString(PyExc_ValueError, e.what()); + } + }); + + py::class_(m, "_OpaqueVmRef"); auto local_m = m.def_submodule("local"); BindLocal(local_m); BindHostSystem(local_m); @@ -379,11 +480,79 @@ void BindLocal(py::module_ &m) { .def("__add__", &local::DeviceAffinity::AddDevice) .def("__repr__", &local::DeviceAffinity::to_s); - py::class_(m, "Program"); + py::class_(m, "Program") + .def(py::new_([](std::span modules, + local::Scope &scope, bool trace_execution) { + local::Program::Options options; + options.trace_execution = trace_execution; + return local::Program::Load(scope.shared_from_this(), modules, + std::move(options)); + }), + py::arg("modules"), py::arg("scope"), py::kw_only(), + py::arg("trace_execution") = false) + .def_prop_ro("exports", &local::Program::exports) + .def("lookup_function", &local::Program::LookupRequiredFunction) + .def("__getitem__", &local::Program::LookupRequiredFunction); + py::class_(m, "ProgramFunction") + .def_prop_ro("name", &local::ProgramFunction::name) + .def_prop_ro("calling_convention", + &local::ProgramFunction::calling_convention) + .def("invocation", &local::ProgramFunction::CreateInvocation, + DOCSTRING_PROGRAM_FUNCTION_INVOCATION) + .def("__call__", PyFunctionCall, py::arg("args")) + .def("__repr__", &local::ProgramFunction::to_s); py::class_(m, "ProgramModule") + .def_prop_ro("exports", &local::ProgramModule::exports) .def("__repr__", &local::ProgramModule::to_s) .def_static("load", &local::ProgramModule::Load, py::arg("system"), py::arg("path"), py::arg("mmap") = true); + py::class_(m, "ProgramInvocation") + .def("invoke", + [](local::ProgramInvocation::Ptr &self) { + if (!self) throw std::invalid_argument("Deallocated invocation"); + return local::ProgramInvocation::Invoke(std::move(self)); + }) + .def("add_arg", + [](local::ProgramInvocation::Ptr &self, py::handle arg) { + if (!self) throw std::invalid_argument("Deallocated invocation"); + py::capsule inv_capsule(self.get()); + PyAddProgramInvocationArg(inv_capsule, arg); + }) + .def("__iter__", + [](local::ProgramInvocation::Ptr &self) { + if (!self) throw std::invalid_argument("Deallocated invocation"); + size_t size = self->results_size(); + py::object tp = py::steal(PyTuple_New(size)); + for (size_t i = 0; i < size; ++i) { + iree::vm_opaque_ref ref = self->result_ref(i); + if (!ref) { + throw new std::logic_error( + "Program returned unsupported Python type"); + } + py::object item = PyRehydrateRef(self.get(), std::move(ref)); + PyTuple_SET_ITEM(tp.ptr(), i, item.release().ptr()); + } + return tp.attr("__iter__")(); + }) + .def( + "__len__", + [](local::ProgramInvocation::Ptr &self) { + if (!self) throw std::invalid_argument("Deallocated invocation"); + return self->results_size(); + }, + "The number of results in this invocation") + .def( + "__getitem__", + [](local::ProgramInvocation::Ptr &self, iree_host_size_t i) { + if (!self) throw std::invalid_argument("Deallocated invocation"); + iree::vm_opaque_ref ref = self->result_ref(i); + if (!ref) { + throw new std::logic_error( + "Program returned unsupported Python type"); + } + return PyRehydrateRef(self.get(), std::move(ref)); + }, + "Gets the i'th result"); struct DevicesSet { DevicesSet(local::Scope &scope) : scope(scope) {} @@ -414,16 +583,7 @@ void BindLocal(py::module_ &m) { [](local::Scope &self, py::args args) { return CastDeviceAffinity(self, args); }, - py::rv_policy::reference_internal) - .def( - "load_unbound_program", - [](local::Scope &scope, std::span modules, - bool trace_execution) { - local::Program::Options options; - options.trace_execution = trace_execution; - return scope.LoadUnboundProgram(modules, std::move(options)); - }, - py::arg("modules"), py::arg("trace_execution") = false); + py::rv_policy::reference_internal); py::class_(m, "ScopedDevice") .def_prop_ro("scope", &local::ScopedDevice::scope, @@ -696,6 +856,20 @@ void BindLocal(py::module_ &m) { return iter_ret; }); py::class_(m, "VoidFuture"); + py::class_( + m, "ProgramInvocationFuture") + .def("result", [](local::ProgramInvocation::Future &self) { + local::ProgramInvocation::Ptr &result = self.result(); + if (!result) return py::none(); + // Sharp edge: ProgramInvocationFutures are read-once since we move the + // ProgramInvocation::Ptr out of the future here and transfer ownership + // to a Python object. There isn't a better way to do this without + // increasing overhead on this hot path or doing something more + // expensive in the C++ API: essentially, ProgramInvocations flow + // through the system precisely one way. As a low level facility, this + // is deemed acceptable. + return py::cast(std::move(result)); + }); py::class_(m, "MessageFuture") .def("result", [](local::MessageFuture &self) { // Get a raw backing msg (without an increased refcount). When cast diff --git a/libshortfin/bindings/python/lib_ext.h b/libshortfin/bindings/python/lib_ext.h index 08cac7b46..f5f3b2733 100644 --- a/libshortfin/bindings/python/lib_ext.h +++ b/libshortfin/bindings/python/lib_ext.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/shortfin/__init__.py b/libshortfin/bindings/python/shortfin/__init__.py index 5426f2ad2..050c2409c 100644 --- a/libshortfin/bindings/python/shortfin/__init__.py +++ b/libshortfin/bindings/python/shortfin/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -14,6 +14,9 @@ Node = _sfl.local.Node Process = _sfl.local.Process Program = _sfl.local.Program +ProgramFunction = _sfl.local.ProgramFunction +ProgramInvocation = _sfl.local.ProgramInvocation +ProgramInvocationFuture = _sfl.local.ProgramInvocationFuture ProgramModule = _sfl.local.ProgramModule Queue = _sfl.local.Queue QueueReader = _sfl.local.QueueReader @@ -37,6 +40,9 @@ "Message", "Node", "Program", + "ProgramFunction", + "ProgramInvocation", + "ProgramInvocationFuture", "ProgramModule", "Queue", "QueueReader", diff --git a/libshortfin/bindings/python/shortfin/amdgpu.py b/libshortfin/bindings/python/shortfin/amdgpu.py index d10dfa459..474d9d7f9 100644 --- a/libshortfin/bindings/python/shortfin/amdgpu.py +++ b/libshortfin/bindings/python/shortfin/amdgpu.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/shortfin/array.py b/libshortfin/bindings/python/shortfin/array.py index 049fe9ed7..d665d280b 100644 --- a/libshortfin/bindings/python/shortfin/array.py +++ b/libshortfin/bindings/python/shortfin/array.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/shortfin/host.py b/libshortfin/bindings/python/shortfin/host.py index 66631fcc4..b3c571297 100644 --- a/libshortfin/bindings/python/shortfin/host.py +++ b/libshortfin/bindings/python/shortfin/host.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/shortfin/interop/fastapi/__init__.py b/libshortfin/bindings/python/shortfin/interop/fastapi/__init__.py index 2cff38342..98f6a41b4 100644 --- a/libshortfin/bindings/python/shortfin/interop/fastapi/__init__.py +++ b/libshortfin/bindings/python/shortfin/interop/fastapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/bindings/python/utils.h b/libshortfin/bindings/python/utils.h index dab4423f8..86a85441f 100644 --- a/libshortfin/bindings/python/utils.h +++ b/libshortfin/bindings/python/utils.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/build_tools/cmake/shortfin_library.cmake b/libshortfin/build_tools/cmake/shortfin_library.cmake index 864751eb7..bae9fe115 100644 --- a/libshortfin/build_tools/cmake/shortfin_library.cmake +++ b/libshortfin/build_tools/cmake/shortfin_library.cmake @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: @@ -24,20 +24,24 @@ function(shortfin_public_library) "COMPONENTS" ${ARGN} ) - shortfin_components_to_static_libs(_STATIC_COMPONENTS ${_RULE_COMPONENTS}) - shortfin_components_to_dynamic_libs(_DYLIB_COMPONENTS ${_RULE_COMPONENTS}) - # Static library. - add_library("${_RULE_NAME}-static" STATIC) - target_link_libraries( - "${_RULE_NAME}-static" PUBLIC ${_STATIC_COMPONENTS} - ) + if(SHORTFIN_BUILD_STATIC) + # Static library. + shortfin_components_to_static_libs(_STATIC_COMPONENTS ${_RULE_COMPONENTS}) + add_library("${_RULE_NAME}-static" STATIC) + target_link_libraries( + "${_RULE_NAME}-static" PUBLIC ${_STATIC_COMPONENTS} + ) + endif() - # Dylib library. - add_library("${_RULE_NAME}" SHARED) - target_compile_definitions("${_RULE_NAME}" INTERFACE _SHORTFIN_USING_DYLIB) - target_link_libraries( - "${_RULE_NAME}" PUBLIC ${_DYLIB_COMPONENTS} - ) + if(SHORTFIN_BUILD_DYNAMIC) + # Dylib library. + shortfin_components_to_dynamic_libs(_DYLIB_COMPONENTS ${_RULE_COMPONENTS}) + add_library("${_RULE_NAME}" SHARED) + target_compile_definitions("${_RULE_NAME}" INTERFACE _SHORTFIN_USING_DYLIB) + target_link_libraries( + "${_RULE_NAME}" PUBLIC ${_DYLIB_COMPONENTS} + ) + endif() endfunction() function(shortfin_cc_component) @@ -48,50 +52,52 @@ function(shortfin_cc_component) "HDRS;SRCS;DEPS;COMPONENTS" ${ARGN} ) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(_STATIC_OBJECTS_NAME "${_RULE_NAME}.objects") - set(_DYLIB_OBJECTS_NAME "${_RULE_NAME}.dylib.objects") - - shortfin_components_to_static_libs(_STATIC_COMPONENTS ${_RULE_COMPONENTS}) - shortfin_components_to_dynamic_libs(_DYLIB_COMPONENTS ${_RULE_COMPONENTS}) - - # Static object library. - add_library(${_STATIC_OBJECTS_NAME} OBJECT) - target_sources(${_STATIC_OBJECTS_NAME} - PRIVATE - ${_RULE_SRCS} - ${_RULE_HDRS} - ) - target_compile_options(${_STATIC_OBJECTS_NAME} PRIVATE ${SHORTFIN_DEFAULT_COPTS}) - target_link_libraries(${_STATIC_OBJECTS_NAME} - PUBLIC - _shortfin_defs - ${_STATIC_COMPONENTS} - ${_RULE_DEPS} - ) + if(SHORTFIN_BUILD_STATIC) + # Static object library. + set(_STATIC_OBJECTS_NAME "${_RULE_NAME}.objects") + shortfin_components_to_static_libs(_STATIC_COMPONENTS ${_RULE_COMPONENTS}) + add_library(${_STATIC_OBJECTS_NAME} OBJECT) + target_sources(${_STATIC_OBJECTS_NAME} + PRIVATE + ${_RULE_SRCS} + ${_RULE_HDRS} + ) + target_compile_options(${_STATIC_OBJECTS_NAME} PRIVATE ${SHORTFIN_DEFAULT_COPTS}) + target_link_libraries(${_STATIC_OBJECTS_NAME} + PUBLIC + _shortfin_defs + ${_STATIC_COMPONENTS} + ${_RULE_DEPS} + ) + endif() - # Dylib object library. - add_library(${_DYLIB_OBJECTS_NAME} OBJECT) - target_sources(${_DYLIB_OBJECTS_NAME} - PRIVATE - ${_RULE_SRCS} - ${_RULE_HDRS} - ) - target_compile_options(${_DYLIB_OBJECTS_NAME} PRIVATE ${SHORTFIN_DEFAULT_COPTS}) - target_link_libraries(${_DYLIB_OBJECTS_NAME} - PUBLIC - _shortfin_defs - ${_DYLIB_COMPONENTS} - ${_RULE_DEPS} - ) - set_target_properties( - ${_DYLIB_OBJECTS_NAME} PROPERTIES - CXX_VISIBILITY_PRESET hidden - C_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON - ) - target_compile_definitions(${_DYLIB_OBJECTS_NAME} - PRIVATE _SHORTFIN_BUILDING_DYLIB) + if(SHORTFIN_BUILD_DYNAMIC) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(_DYLIB_OBJECTS_NAME "${_RULE_NAME}.dylib.objects") + shortfin_components_to_dynamic_libs(_DYLIB_COMPONENTS ${_RULE_COMPONENTS}) + # Dylib object library. + add_library(${_DYLIB_OBJECTS_NAME} OBJECT) + target_sources(${_DYLIB_OBJECTS_NAME} + PRIVATE + ${_RULE_SRCS} + ${_RULE_HDRS} + ) + target_compile_options(${_DYLIB_OBJECTS_NAME} PRIVATE ${SHORTFIN_DEFAULT_COPTS}) + target_link_libraries(${_DYLIB_OBJECTS_NAME} + PUBLIC + _shortfin_defs + ${_DYLIB_COMPONENTS} + ${_RULE_DEPS} + ) + set_target_properties( + ${_DYLIB_OBJECTS_NAME} PROPERTIES + CXX_VISIBILITY_PRESET hidden + C_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + ) + target_compile_definitions(${_DYLIB_OBJECTS_NAME} + PRIVATE _SHORTFIN_BUILDING_DYLIB) + endif() endfunction() function(shortfin_components_to_static_libs out_static_libs) @@ -122,7 +128,7 @@ function(shortfin_gtest_test) add_executable(${_RULE_NAME} ${_RULE_SRCS}) target_link_libraries(${_RULE_NAME} PRIVATE ${_RULE_DEPS} - shortfin + ${SHORTFIN_LINK_LIBRARY_NAME} GTest::gmock GTest::gtest_main ) diff --git a/libshortfin/build_tools/python_lsan_suppressions.txt b/libshortfin/build_tools/python_lsan_suppressions.txt new file mode 100644 index 000000000..cc768d575 --- /dev/null +++ b/libshortfin/build_tools/python_lsan_suppressions.txt @@ -0,0 +1,2 @@ +leak:PyUnicode_New +leak:_PyUnicodeWriter_Finish diff --git a/libshortfin/examples/python/async/basic_asyncio.py b/libshortfin/examples/python/async/basic_asyncio.py index 256f8f380..d14ffb613 100644 --- a/libshortfin/examples/python/async/basic_asyncio.py +++ b/libshortfin/examples/python/async/basic_asyncio.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/async/device_sync.py b/libshortfin/examples/python/async/device_sync.py index 5b43b58fb..b88049ae0 100644 --- a/libshortfin/examples/python/async/device_sync.py +++ b/libshortfin/examples/python/async/device_sync.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/async/process.py b/libshortfin/examples/python/async/process.py index a523efc58..dcf539516 100644 --- a/libshortfin/examples/python/async/process.py +++ b/libshortfin/examples/python/async/process.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/async/queue.py b/libshortfin/examples/python/async/queue.py index 47b3fb64c..dabc2c7f0 100644 --- a/libshortfin/examples/python/async/queue.py +++ b/libshortfin/examples/python/async/queue.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/fastapi/server.py b/libshortfin/examples/python/fastapi/server.py index 66ab37b75..639cf2d00 100644 --- a/libshortfin/examples/python/fastapi/server.py +++ b/libshortfin/examples/python/fastapi/server.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/mobilenet_server/build_model.sh b/libshortfin/examples/python/mobilenet_server/build_model.sh index e059e344a..6011072c9 100755 --- a/libshortfin/examples/python/mobilenet_server/build_model.sh +++ b/libshortfin/examples/python/mobilenet_server/build_model.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/examples/python/mobilenet_server/inference_system.py b/libshortfin/examples/python/mobilenet_server/inference_system.py index 8ae7773db..068204f49 100644 --- a/libshortfin/examples/python/mobilenet_server/inference_system.py +++ b/libshortfin/examples/python/mobilenet_server/inference_system.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -24,7 +24,7 @@ def __init__(self, raw_image_data): class InferenceProcess(sf.Process): def __init__(self, program, request_queue, **kwargs): super().__init__(**kwargs) - self.program = program + self.main_function = program["module.torch-jit-export"] self.request_reader = request_queue.reader() self.device = self.scope.device(0) self.device_input = sfnp.device_array( @@ -41,14 +41,46 @@ async def run(self): # support for. Generally, APIs on storage should be mirrored onto # the array. self.host_staging.storage.data = request.raw_image_data - print(self.host_staging) - self.device_input.storage.copy_from(self.host_staging.storage) - print(self.device_input) + print("host_staging =", self.host_staging) + self.device_input.copy_from(self.host_staging) + + # Simple call. Note that the await here is merely awaiting the + # result being *available* (i.e. that the VM coroutine has + # completed) but does not indicate that the result is ready. + (result1,) = await self.main_function(self.device_input) + (result2,) = await self.main_function(self.device_input) + + # TODO: Implement await on individual results. The accounting is + # there but currently we can only await on the device itself. + await self.device + print("Result 1:", result1) + print("Result 2:", result2) + + # Explicit invocation object. + # inv = self.main_function.invocation(scope=self.scope) + # inv.add_arg(self.device_input) + # results = await inv.invoke() + # print("results:", results) + + # Multiple invocations in parallel. + # all_results = await asyncio.gather( + # self.main_function(self.device_input, scope=self.scope), + # self.main_function(self.device_input, scope=self.scope), + # self.main_function(self.device_input, scope=self.scope), + # ) + # print("All results:", all_results) + + # output = await self.scope.invoke(self.main_function, self.device_input) + # print("OUTPUT:", output) + # read_back = self.device_input.for_transfer() + # read_back.copy_from(self.device_input) + # await self.device + # print("read back =", read_back) class Main: def __init__(self, lsys: sf.System, home_dir: Path): - self.processes_per_worker = 1 + self.processes_per_worker = 2 self.lsys = lsys self.home_dir = home_dir self.request_queue = lsys.create_queue("request") @@ -60,8 +92,8 @@ async def start_scope(self, scope): # Note that currently, program load is synchronous. But we do it # in a task so we can await it in the future and let program loads # overlap. - program = scope.load_unbound_program([self.program_module]) for _ in range(self.processes_per_worker): + program = sf.Program([self.program_module], scope=scope) self.processes.append( InferenceProcess(program, self.request_queue, scope=scope).launch() ) diff --git a/libshortfin/examples/python/mobilenet_server/upgrade_onnx.py b/libshortfin/examples/python/mobilenet_server/upgrade_onnx.py index 2b85309ff..9b31d449f 100644 --- a/libshortfin/examples/python/mobilenet_server/upgrade_onnx.py +++ b/libshortfin/examples/python/mobilenet_server/upgrade_onnx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/pyproject.toml b/libshortfin/pyproject.toml index e868b4264..f1b34c64a 100644 --- a/libshortfin/pyproject.toml +++ b/libshortfin/pyproject.toml @@ -3,7 +3,6 @@ requires = [ "cmake>=3.29", "setuptools>=61.0", "wheel", - "nanobind>=2.0", "ninja", ] build-backend = "setuptools.build_meta" diff --git a/libshortfin/requirements-iree-compiler.txt b/libshortfin/requirements-iree-compiler.txt new file mode 100644 index 000000000..34228ccf4 --- /dev/null +++ b/libshortfin/requirements-iree-compiler.txt @@ -0,0 +1,3 @@ +# Keep in sync with IREE_REF in CI and GIT_TAG in CMakeLists.txt +-f https://iree.dev/pip-release-links.html +iree-compiler==20240904.1006 diff --git a/libshortfin/requirements-tests.txt b/libshortfin/requirements-tests.txt index 50bdd9831..1049b0412 100644 --- a/libshortfin/requirements-tests.txt +++ b/libshortfin/requirements-tests.txt @@ -1,4 +1,3 @@ -nanobind==2.0.0 pytest requests fastapi diff --git a/libshortfin/setup.py b/libshortfin/setup.py index 4f4074d2f..b28b8d114 100644 --- a/libshortfin/setup.py +++ b/libshortfin/setup.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/CMakeLists.txt b/libshortfin/src/CMakeLists.txt index de31643e3..51dd5f87f 100644 --- a/libshortfin/src/CMakeLists.txt +++ b/libshortfin/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: diff --git a/libshortfin/src/shortfin/CMakeLists.txt b/libshortfin/src/shortfin/CMakeLists.txt index 3c50ed1eb..1bea0003b 100644 --- a/libshortfin/src/shortfin/CMakeLists.txt +++ b/libshortfin/src/shortfin/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: diff --git a/libshortfin/src/shortfin/array/CMakeLists.txt b/libshortfin/src/shortfin/array/CMakeLists.txt index 0e9360363..d40eed23f 100644 --- a/libshortfin/src/shortfin/array/CMakeLists.txt +++ b/libshortfin/src/shortfin/array/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: @@ -12,6 +12,7 @@ shortfin_cc_component( api.h dims.h dtype.h + dtypes.inl storage.h SRCS array.cc diff --git a/libshortfin/src/shortfin/array/api.h b/libshortfin/src/shortfin/array/api.h index baa8a55ea..d425e4fc5 100644 --- a/libshortfin/src/shortfin/array/api.h +++ b/libshortfin/src/shortfin/array/api.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/array/array.cc b/libshortfin/src/shortfin/array/array.cc index 1d6d7cc5a..9a0b22bf2 100644 --- a/libshortfin/src/shortfin/array/array.cc +++ b/libshortfin/src/shortfin/array/array.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -14,25 +14,36 @@ namespace shortfin::array { -template class InlinedDims; +template class InlinedDims; // -------------------------------------------------------------------------- // // device_array // -------------------------------------------------------------------------- // -const mapping device_array::data() const { return storage_.MapRead(); } +device_array::device_array(class storage storage, + std::span shape, DType dtype) + : base_array(shape, dtype), storage_(std::move(storage)) { + auto needed_size = this->dtype().compute_dense_nd_size(this->shape()); + if (storage_.byte_length() < needed_size) { + throw std::invalid_argument( + fmt::format("Array storage requires at least {} bytes but has only {}", + needed_size, storage_.byte_length())); + } +} + +const mapping device_array::data() const { return storage_.map_read(); } -mapping device_array::data() { return storage_.MapRead(); } +mapping device_array::data() { return storage_.map_read(); } -mapping device_array::data_rw() { return storage_.MapReadWrite(); } +mapping device_array::data_rw() { return storage_.map_read_write(); } -mapping device_array::data_w() { return storage_.MapWriteDiscard(); } +mapping device_array::data_w() { return storage_.map_write_discard(); } std::optional device_array::map_memory_for_xtensor() { if (storage_.is_mappable_for_read_write()) { - return storage_.MapReadWrite(); + return storage_.map_read_write(); } else if (storage_.is_mappable_for_read()) { - return storage_.MapRead(); + return storage_.map_read(); } return {}; } @@ -52,10 +63,49 @@ std::string device_array::to_s() const { } } - return fmt::format("device_array([{}], dtype='{}', device={}({})) ={}{}", - fmt::join(shape(), ", "), dtype().name(), - storage_.device().to_s(), storage_.formatted_memory_type(), - contents_prefix, contents); + return fmt::format( + "device_array([{}], dtype='{}', device={}(type={}, usage={}, access={})) " + "={}{}", + fmt::join(shape(), ", "), dtype().name(), storage_.device().to_s(), + storage_.formatted_memory_type(), storage_.formatted_buffer_usage(), + storage_.formatted_memory_access(), contents_prefix, contents); +} + +void device_array::AddAsInvocationArgument( + local::ProgramInvocation *inv, local::ProgramResourceBarrier barrier) { + auto dims_span = shape(); + iree_hal_buffer_view_t *buffer_view; + SHORTFIN_THROW_IF_ERROR(iree_hal_buffer_view_create( + storage_, dims_span.size(), dims_span.data(), dtype(), + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, storage_.host_allocator(), + &buffer_view)); + + iree::vm_opaque_ref ref; + *(&ref) = iree_hal_buffer_view_move_ref(buffer_view); + inv->AddArg(std::move(ref)); + + storage().AddInvocationArgBarrier(inv, barrier); +} + +iree_vm_ref_type_t device_array::invocation_marshalable_type() { + return iree_hal_buffer_view_type(); +} + +device_array device_array::CreateFromInvocationResultRef( + local::ProgramInvocation *inv, iree::vm_opaque_ref ref) { + // We don't retain the buffer view in the device array, so just deref it + // vs stealing the ref. + iree_hal_buffer_view_t *bv = iree_hal_buffer_view_deref(*ref.get()); + iree::hal_buffer_ptr buffer = + iree::hal_buffer_ptr::borrow_reference(iree_hal_buffer_view_buffer(bv)); + + auto imported_storage = + storage::ImportInvocationResultStorage(inv, std::move(buffer)); + std::span shape(iree_hal_buffer_view_shape_dims(bv), + iree_hal_buffer_view_shape_rank(bv)); + return device_array( + std::move(imported_storage), shape, + DType::import_element_type(iree_hal_buffer_view_element_type(bv))); } } // namespace shortfin::array diff --git a/libshortfin/src/shortfin/array/array.h b/libshortfin/src/shortfin/array/array.h index c3ab6e302..875155757 100644 --- a/libshortfin/src/shortfin/array/array.h +++ b/libshortfin/src/shortfin/array/array.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -16,6 +16,7 @@ #include "shortfin/array/dtype.h" #include "shortfin/array/storage.h" #include "shortfin/array/xtensor_bridge.h" +#include "shortfin/local/program_interfaces.h" #include "shortfin/support/api.h" namespace shortfin::array { @@ -23,7 +24,8 @@ namespace shortfin::array { // Either a host or device nd-array view. class SHORTFIN_API base_array { public: - base_array(std::span shape, DType dtype) : dtype_(dtype) { + base_array(std::span shape, DType dtype) + : dtype_(dtype) { set_shape(shape); } // Need to explicitly define copy/move constructors even though this is @@ -38,9 +40,9 @@ class SHORTFIN_API base_array { DType dtype() const { return dtype_; } // Access shape. - void set_shape(std::span shape) { shape_.set(shape); } - std::span shape() const { return shape_.span(); } - std::span mutable_shape() { return shape_.span(); } + void set_shape(std::span shape) { shape_.set(shape); } + std::span shape() const { return shape_.span(); } + std::span mutable_shape() { return shape_.span(); } // Sometimes we need to access the raw shape container (i.e. for adapters, // etc). @@ -54,29 +56,31 @@ class SHORTFIN_API base_array { class SHORTFIN_API device_array : public base_array, - public poly_xt_mixin { + public poly_xt_mixin, + public local::ProgramInvocationMarshalable { public: - device_array(class storage storage, std::span shape, - DType dtype) - : base_array(shape, dtype), storage_(std::move(storage)) {} + device_array(class storage storage, std::span shape, + DType dtype); class storage &storage() { return storage_; } local::ScopedDevice &device() { return storage_.device(); } // Allocate an array on the device. static device_array for_device(local::ScopedDevice &device, - std::span shape, DType dtype) { + std::span shape, + DType dtype) { return device_array( - storage::AllocateDevice(device, dtype.compute_dense_nd_size(shape)), + storage::allocate_device(device, dtype.compute_dense_nd_size(shape)), shape, dtype); } // Allocates a host array that is registered by the device. This can include // arrays that are visible from different combinations of host/device. static device_array for_host(local::ScopedDevice &device, - std::span shape, DType dtype) { + std::span shape, + DType dtype) { return device_array( - storage::AllocateHost(device, dtype.compute_dense_nd_size(shape)), + storage::allocate_host(device, dtype.compute_dense_nd_size(shape)), shape, dtype); } @@ -85,6 +89,23 @@ class SHORTFIN_API device_array return for_host(storage().device(), shape(), dtype()); } + // Enqueues a fill of the storage with an arbitrary pattern of the given + // size. The pattern size must be 1, 2, or 4. Equivalent to calling the same + // on the backing storage. + void fill(const void *pattern, iree_host_size_t pattern_length) { + storage_.fill(pattern, pattern_length); + } + + // Performs either a d2h, h2d or d2d transfer from a source storage to this + // storage. Equivalent to calling the same on the backing storage. + void copy_from(device_array &source_array) { + storage_.copy_from(source_array.storage_); + } + // Inverse of copy_from. + void copy_to(device_array &dest_array) { + dest_array.storage_.copy_from(storage_); + } + // Untyped access to the backing data. The array must be mappable. Specific // access modes: // * data(): Read-only access to the data. @@ -123,6 +144,15 @@ class SHORTFIN_API device_array protected: class storage storage_; + + private: + // ProgramInvocationMarshalable implementation. + void AddAsInvocationArgument(local::ProgramInvocation *inv, + local::ProgramResourceBarrier barrier) override; + static device_array CreateFromInvocationResultRef( + local::ProgramInvocation *inv, iree::vm_opaque_ref ref); + static iree_vm_ref_type_t invocation_marshalable_type(); + friend class shortfin::local::ProgramInvocationMarshalableFactory; }; } // namespace shortfin::array diff --git a/libshortfin/src/shortfin/array/array_test.cc b/libshortfin/src/shortfin/array/array_test.cc index 2c435b292..50d9c2b00 100644 --- a/libshortfin/src/shortfin/array/array_test.cc +++ b/libshortfin/src/shortfin/array/array_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/array/dims.h b/libshortfin/src/shortfin/array/dims.h index 529aebc42..a0cbacd00 100644 --- a/libshortfin/src/shortfin/array/dims.h +++ b/libshortfin/src/shortfin/array/dims.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -11,6 +11,7 @@ #include #include +#include "iree/hal/buffer_view.h" #include "shortfin/support/api.h" namespace shortfin::array { @@ -248,8 +249,8 @@ class SHORTFIN_API InlinedDims { _D dims_; }; -extern template class InlinedDims; -using Dims = InlinedDims; +extern template class InlinedDims; +using Dims = InlinedDims; } // namespace shortfin::array diff --git a/libshortfin/src/shortfin/array/dims_test.cc b/libshortfin/src/shortfin/array/dims_test.cc index 287e2fa9a..926545fec 100644 --- a/libshortfin/src/shortfin/array/dims_test.cc +++ b/libshortfin/src/shortfin/array/dims_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/array/dtype.cc b/libshortfin/src/shortfin/array/dtype.cc index e6b92a7e9..3b5e6bfe9 100644 --- a/libshortfin/src/shortfin/array/dtype.cc +++ b/libshortfin/src/shortfin/array/dtype.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,6 +6,8 @@ #include "shortfin/array/dtype.h" +#include + #include "fmt/core.h" namespace shortfin::array { @@ -22,4 +24,21 @@ iree_device_size_t DType::compute_dense_nd_size(std::span dims) { return accum; } +DType DType::import_element_type(iree_hal_element_type_t et) { + static std::unordered_map static_canonical = + ([]() { + std::unordered_map c; + auto add = [&](DType dt) { c.emplace(std::make_pair(dt.et_, dt)); }; +#define SHORTFIN_DTYPE_HANDLE(et, ident) add(DType(et, #ident)); +#include "shortfin/array/dtypes.inl" +#undef SHORTFIN_DTYPE_HANDLE + return c; + })(); + + auto &c = static_canonical; + auto it = c.find(et); + if (it != c.end()) return it->second; + return DType(et, "opaque_imported"); +} + } // namespace shortfin::array diff --git a/libshortfin/src/shortfin/array/dtype.h b/libshortfin/src/shortfin/array/dtype.h index 090751162..061122ae7 100644 --- a/libshortfin/src/shortfin/array/dtype.h +++ b/libshortfin/src/shortfin/array/dtype.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -19,64 +19,10 @@ namespace shortfin::array { // Wraps an iree_hal_element_type into a DType like object. class SHORTFIN_API DType { public: - static DType opaque8() { - return DType(IREE_HAL_ELEMENT_TYPE_OPAQUE_8, "opaque8"); - } - static DType opaque16() { - return DType(IREE_HAL_ELEMENT_TYPE_OPAQUE_16, "opaque16"); - } - static DType opaque32() { - return DType(IREE_HAL_ELEMENT_TYPE_OPAQUE_32, "opaque32"); - } - static DType opaque64() { - return DType(IREE_HAL_ELEMENT_TYPE_OPAQUE_64, "opaque64"); - } - static DType bool8() { return DType(IREE_HAL_ELEMENT_TYPE_BOOL_8, "bool8"); } - static DType int4() { return DType(IREE_HAL_ELEMENT_TYPE_INT_4, "int4"); } - static DType sint4() { return DType(IREE_HAL_ELEMENT_TYPE_SINT_4, "sint4"); } - static DType uint4() { return DType(IREE_HAL_ELEMENT_TYPE_UINT_4, "uint4"); } - static DType int8() { return DType(IREE_HAL_ELEMENT_TYPE_INT_8, "int8"); } - static DType sint8() { return DType(IREE_HAL_ELEMENT_TYPE_SINT_8, "sint8"); } - static DType uint8() { return DType(IREE_HAL_ELEMENT_TYPE_UINT_8, "uint8"); } - static DType int16() { return DType(IREE_HAL_ELEMENT_TYPE_INT_16, "int16"); } - static DType sint16() { - return DType(IREE_HAL_ELEMENT_TYPE_SINT_16, "sint16"); - } - static DType uint16() { - return DType(IREE_HAL_ELEMENT_TYPE_UINT_16, "uint16"); - } - static DType int32() { return DType(IREE_HAL_ELEMENT_TYPE_INT_32, "int32"); } - static DType sint32() { - return DType(IREE_HAL_ELEMENT_TYPE_SINT_32, "sint32"); - } - static DType uint32() { - return DType(IREE_HAL_ELEMENT_TYPE_UINT_32, "uint32"); - } - static DType int64() { return DType(IREE_HAL_ELEMENT_TYPE_INT_64, "int64"); } - static DType sint64() { - return DType(IREE_HAL_ELEMENT_TYPE_SINT_64, "sint64"); - } - static DType uint64() { - return DType(IREE_HAL_ELEMENT_TYPE_UINT_64, "uint64"); - } - static DType float16() { - return DType(IREE_HAL_ELEMENT_TYPE_FLOAT_16, "float16"); - } - static DType float32() { - return DType(IREE_HAL_ELEMENT_TYPE_FLOAT_32, "float32"); - } - static DType float64() { - return DType(IREE_HAL_ELEMENT_TYPE_FLOAT_64, "float64"); - } - static DType bfloat16() { - return DType(IREE_HAL_ELEMENT_TYPE_BFLOAT_16, "bfloat16"); - } - static DType complex64() { - return DType(IREE_HAL_ELEMENT_TYPE_COMPLEX_FLOAT_64, "complex64"); - } - static DType complex128() { - return DType(IREE_HAL_ELEMENT_TYPE_COMPLEX_FLOAT_128, "complex128"); - } +#define SHORTFIN_DTYPE_HANDLE(et, ident) \ + static DType ident() { return DType(et, #ident); } +#include "shortfin/array/dtypes.inl" +#undef SHORTFIN_DTYPE_HANDLE operator iree_hal_element_type_t() const { return et_; } @@ -112,6 +58,9 @@ class SHORTFIN_API DType { bool operator==(const DType &other) const { return et_ == other.et_; } + // Imports a raw iree_hal_element_type_t from the ether. + static DType import_element_type(iree_hal_element_type_t et); + private: DType(iree_hal_element_type_t et, std::string_view name) : et_(et), name_(name) {} diff --git a/libshortfin/src/shortfin/array/dtype_test.cc b/libshortfin/src/shortfin/array/dtype_test.cc index f1dc0477a..d5f38c974 100644 --- a/libshortfin/src/shortfin/array/dtype_test.cc +++ b/libshortfin/src/shortfin/array/dtype_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/array/dtypes.inl b/libshortfin/src/shortfin/array/dtypes.inl new file mode 100644 index 000000000..20d93e5d7 --- /dev/null +++ b/libshortfin/src/shortfin/array/dtypes.inl @@ -0,0 +1,34 @@ +// Copyright 2024 Advanced Micro Devices, Inc. +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Include file API for enumerating all known dtypes. + +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_OPAQUE_8, opaque8) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_OPAQUE_16, opaque16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_OPAQUE_32, opaque32) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_OPAQUE_64, opaque64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_BOOL_8, bool8) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_INT_4, int4) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_SINT_4, sint4) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_UINT_4, uint4) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_INT_8, int8) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_SINT_8, sint8) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_UINT_8, uint8) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_INT_16, int16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_SINT_16, sint16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_UINT_16, uint16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_INT_32, int32) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_SINT_32, sint32) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_UINT_32, uint32) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_INT_64, int64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_SINT_64, sint64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_UINT_64, uint64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_FLOAT_16, float16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_FLOAT_32, float32) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_FLOAT_64, float64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_BFLOAT_16, bfloat16) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_COMPLEX_FLOAT_64, complex64) +SHORTFIN_DTYPE_HANDLE(IREE_HAL_ELEMENT_TYPE_COMPLEX_FLOAT_128, complex128) diff --git a/libshortfin/src/shortfin/array/storage.cc b/libshortfin/src/shortfin/array/storage.cc index fa9e0f4b8..542f69025 100644 --- a/libshortfin/src/shortfin/array/storage.cc +++ b/libshortfin/src/shortfin/array/storage.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -35,8 +35,14 @@ storage::storage(local::ScopedDevice device, iree::hal_buffer_ptr buffer, } storage::~storage() { logging::destruct("array::storage", this); } -storage storage::AllocateDevice(ScopedDevice &device, - iree_device_size_t allocation_size) { +storage storage::import_buffer(local::ScopedDevice &device, + iree::hal_buffer_ptr buffer) { + return storage(device, std::move(buffer), + device.scope().NewTimelineResource()); +} + +storage storage::allocate_device(ScopedDevice &device, + iree_device_size_t allocation_size) { if (!device.raw_device()) { throw std::invalid_argument("Cannot allocate with a null device affinity"); } @@ -54,8 +60,8 @@ storage storage::AllocateDevice(ScopedDevice &device, device.scope().NewTimelineResource()); } -storage storage::AllocateHost(ScopedDevice &device, - iree_device_size_t allocation_size) { +storage storage::allocate_host(ScopedDevice &device, + iree_device_size_t allocation_size) { if (!device.raw_device()) { throw std::invalid_argument("Cannot allocate with a null device affinity"); } @@ -64,7 +70,8 @@ storage storage::AllocateHost(ScopedDevice &device, iree_hal_buffer_params_t params = { .usage = IREE_HAL_BUFFER_USAGE_MAPPING, .access = IREE_HAL_MEMORY_ACCESS_ALL, - .type = IREE_HAL_MEMORY_TYPE_OPTIMAL_FOR_HOST, + .type = IREE_HAL_MEMORY_TYPE_OPTIMAL_FOR_HOST | + IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, .queue_affinity = device.affinity().queue_affinity(), }; if (device.affinity().queue_affinity() != 0) { @@ -76,7 +83,7 @@ storage storage::AllocateHost(ScopedDevice &device, device.scope().NewTimelineResource()); } -storage storage::Subspan(iree_device_size_t byte_offset, +storage storage::subspan(iree_device_size_t byte_offset, iree_device_size_t byte_length) { storage new_storage(device_, {}, timeline_resource_); SHORTFIN_THROW_IF_ERROR(iree_hal_buffer_subspan( @@ -84,7 +91,7 @@ storage storage::Subspan(iree_device_size_t byte_offset, return new_storage; } -void storage::Fill(const void *pattern, iree_host_size_t pattern_length) { +void storage::fill(const void *pattern, iree_host_size_t pattern_length) { device_.scope().scheduler().AppendCommandBuffer( device_, TransactionType::TRANSFER, [&](Account &account) { // Must depend on all of this buffer's use dependencies to avoid @@ -94,9 +101,8 @@ void storage::Fill(const void *pattern, iree_host_size_t pattern_length) { // write-after-write hazard. account.active_deps_extend(timeline_resource_->mutation_barrier()); - // TODO: I need to join the submission dependencies on the account - // with the timeline resource idle fence to ensure that - // write-after-access is properly sequenced. + SHORTFIN_SCHED_LOG(" : FillBuffer({})", + static_cast(buffer_.get())); SHORTFIN_THROW_IF_ERROR(iree_hal_command_buffer_fill_buffer( account.active_command_buffer(), iree_hal_make_buffer_ref( @@ -111,7 +117,7 @@ void storage::Fill(const void *pattern, iree_host_size_t pattern_length) { }); } -void storage::CopyFrom(storage &source_storage) { +void storage::copy_from(storage &source_storage) { device_.scope().scheduler().AppendCommandBuffer( device_, TransactionType::TRANSFER, [&](Account &account) { // Must depend on the source's mutation dependencies to avoid @@ -122,6 +128,9 @@ void storage::CopyFrom(storage &source_storage) { account.active_deps_extend(timeline_resource_->use_barrier()); account.active_deps_extend(timeline_resource_->mutation_barrier()); + SHORTFIN_SCHED_LOG(" : CopyBuffer({} -> {})", + static_cast(source_storage.buffer_.get()), + static_cast(buffer_.get())); SHORTFIN_THROW_IF_ERROR(iree_hal_command_buffer_copy_buffer( account.active_command_buffer(), /*source_ref=*/ @@ -129,10 +138,13 @@ void storage::CopyFrom(storage &source_storage) { /*target_ref=*/ iree_hal_make_buffer_ref(buffer_, 0, byte_length()))); - // And move our own mutation barrier to the current pending timeline + // Move our own mutation barrier to the current pending timeline // value. timeline_resource_->set_mutation_barrier( account.timeline_sem(), account.timeline_idle_timepoint()); + // And extend the source use barrier. + source_storage.timeline_resource_->use_barrier_insert( + account.timeline_sem(), account.timeline_idle_timepoint()); }); } @@ -150,7 +162,7 @@ bool storage::is_mappable_for_read_write() const { (IREE_HAL_MEMORY_ACCESS_READ | IREE_HAL_MEMORY_ACCESS_WRITE)); } -void storage::MapExplicit(mapping &mapping, iree_hal_memory_access_t access) { +void storage::map_explicit(mapping &mapping, iree_hal_memory_access_t access) { assert(access != IREE_HAL_MEMORY_ACCESS_NONE); mapping.reset(); SHORTFIN_THROW_IF_ERROR(iree_hal_buffer_map_range( @@ -189,6 +201,68 @@ std::string storage::formatted_buffer_usage() const { return std::string(sv.data, sv.size); } +void storage::AddAsInvocationArgument(local::ProgramInvocation *inv, + local::ProgramResourceBarrier barrier) { + iree::vm_opaque_ref ref; + *(&ref) = iree_hal_buffer_retain_ref(buffer_); + inv->AddArg(std::move(ref)); + + AddInvocationArgBarrier(inv, barrier); +} + +iree_vm_ref_type_t storage::invocation_marshalable_type() { + return iree_hal_buffer_type(); +} + +storage storage::CreateFromInvocationResultRef(local::ProgramInvocation *inv, + iree::vm_opaque_ref ref) { + // Steal the ref to one of our smart pointers. + // TODO: Should have an opaque_ref::release(). + iree::hal_buffer_ptr buffer = + iree::hal_buffer_ptr::steal_reference(iree_hal_buffer_deref(*ref.get())); + (&ref)->ptr = nullptr; + return ImportInvocationResultStorage(inv, std::move(buffer)); +} + +storage storage::ImportInvocationResultStorage(local::ProgramInvocation *inv, + iree::hal_buffer_ptr buffer) { + local::ScopedDevice device = + local::ScopedDevice(*inv->scope(), inv->device_selection()); + auto imported_storage = storage::import_buffer(device, std::move(buffer)); + + auto coarse_signal = inv->coarse_signal(); + if (coarse_signal.first) { + SHORTFIN_SCHED_LOG("Storage buffer {}: Ready barrier {}@{}", + static_cast(imported_storage.buffer_.get()), + static_cast(coarse_signal.first), + coarse_signal.second); + imported_storage.timeline_resource_->set_mutation_barrier( + coarse_signal.first, coarse_signal.second); + imported_storage.timeline_resource_->use_barrier_insert( + coarse_signal.first, coarse_signal.second); + } + + return imported_storage; +} + +void storage::AddInvocationArgBarrier(local::ProgramInvocation *inv, + local::ProgramResourceBarrier barrier) { + switch (barrier) { + case ProgramResourceBarrier::DEFAULT: + case ProgramResourceBarrier::READ: + inv->wait_insert(timeline_resource_->mutation_barrier()); + inv->DeviceSelect(device_.affinity()); + break; + case ProgramResourceBarrier::WRITE: + inv->wait_insert(timeline_resource_->mutation_barrier()); + inv->wait_insert(timeline_resource_->use_barrier()); + inv->DeviceSelect(device_.affinity()); + break; + case ProgramResourceBarrier::NONE: + break; + } +} + std::string storage::to_s() const { return fmt::format("", static_cast(buffer_.get()), byte_length()); diff --git a/libshortfin/src/shortfin/array/storage.h b/libshortfin/src/shortfin/array/storage.h index 0db73d28f..5f3a568b2 100644 --- a/libshortfin/src/shortfin/array/storage.h +++ b/libshortfin/src/shortfin/array/storage.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,6 +9,7 @@ #include +#include "shortfin/local/program_interfaces.h" #include "shortfin/local/scope.h" #include "shortfin/support/api.h" @@ -70,7 +71,7 @@ class SHORTFIN_API mapping { }; // Array storage backed by an IREE buffer of some form. -class SHORTFIN_API storage { +class SHORTFIN_API storage : public local::ProgramInvocationMarshalable { public: ~storage(); local::ScopedDevice &device() { return device_; } @@ -78,32 +79,35 @@ class SHORTFIN_API storage { const local::ScopedDevice &device() const { return device_; } local::Scope &scope() const { return device_.scope(); } + static storage import_buffer(local::ScopedDevice &device, + iree::hal_buffer_ptr buffer); + // Allocates device storage, compatible with the given device affinity. // By default, this will be IREE_HAL_MEMORY_TYPE_OPTIMAL_FOR_DEVICE. - static storage AllocateDevice(local::ScopedDevice &device, - iree_device_size_t allocation_size); + static storage allocate_device(local::ScopedDevice &device, + iree_device_size_t allocation_size); // Allocates host storage, compatible with the given device affinity. // By default, if there are any affinity bits set in the device, then // the storage will be device visible and have permitted usage for // transfers. This default policy can be overriden based on device defaults // or explicit options. - static storage AllocateHost(local::ScopedDevice &device, - iree_device_size_t allocation_size); + static storage allocate_host(local::ScopedDevice &device, + iree_device_size_t allocation_size); // Creates a subspan view of the current storage given a byte offset and // length. The returned storage shares the underlying allocation and // scheduling control block. - storage Subspan(iree_device_size_t byte_offset, + storage subspan(iree_device_size_t byte_offset, iree_device_size_t byte_length); // Enqueues a fill of the storage with an arbitrary pattern of the given // size. The pattern size must be 1, 2, or 4. - void Fill(const void *pattern, iree_host_size_t pattern_length); + void fill(const void *pattern, iree_host_size_t pattern_length); // Performs either a d2h, h2d or d2d transfer from a source storage to this // storage. - void CopyFrom(storage &source_storage); + void copy_from(storage &source_storage); iree_device_size_t byte_length() const { return iree_hal_buffer_byte_length(buffer_.get()); @@ -124,52 +128,80 @@ class SHORTFIN_API storage { bool is_mappable_for_read_write() const; // Maps the memory for access from a host pointer using a scoped mapping. - void MapExplicit(mapping &mapping, iree_hal_memory_access_t access); + void map_explicit(mapping &mapping, iree_hal_memory_access_t access); // Maps the memory for read/write access, preserving any contents. - mapping MapReadWrite() { + mapping map_read_write() { mapping m; - MapExplicit(m, IREE_HAL_MEMORY_ACCESS_READ | IREE_HAL_MEMORY_ACCESS_WRITE); + map_explicit(m, IREE_HAL_MEMORY_ACCESS_READ | IREE_HAL_MEMORY_ACCESS_WRITE); return m; } // Maps the memory for discard write. This is used if populating an initial // buffer. - mapping MapWriteDiscard() { + mapping map_write_discard() { mapping m; - MapExplicit(m, IREE_HAL_MEMORY_ACCESS_DISCARD_WRITE); + map_explicit(m, IREE_HAL_MEMORY_ACCESS_DISCARD_WRITE); return m; } // Maps the memory for read-only access. - mapping MapRead() { + mapping map_read() { mapping m; - MapExplicit(m, IREE_HAL_MEMORY_ACCESS_READ); + map_explicit(m, IREE_HAL_MEMORY_ACCESS_READ); return m; } - const mapping MapRead() const { + const mapping map_read() const { mapping m; - const_cast(this)->MapExplicit(m, IREE_HAL_MEMORY_ACCESS_READ); + const_cast(this)->map_explicit(m, IREE_HAL_MEMORY_ACCESS_READ); return m; } std::string to_s() const; + bool operator==(const storage &other) const { + return other.buffer_.get() == buffer_.get(); + } + // Access raw buffer. This must not be retained apart from the storage for // any length of time that may extend its lifetime (as the storage keeps // underlying device references alive as needed). operator iree_hal_buffer_t *() { return buffer_; } + iree_allocator_t host_allocator() { + return timeline_resource_->host_allocator(); + } + private: storage(local::ScopedDevice device, iree::hal_buffer_ptr buffer, local::detail::TimelineResource::Ref timeline_resource); + // ProgramInvocationMarshalable implementation. + void AddAsInvocationArgument(local::ProgramInvocation *inv, + local::ProgramResourceBarrier barrier) override; + static storage CreateFromInvocationResultRef(local::ProgramInvocation *inv, + iree::vm_opaque_ref ref); + static iree_vm_ref_type_t invocation_marshalable_type(); + + // Adds any necessary wait barriers to the invocation on behalf of this + // storage. + void AddInvocationArgBarrier(local::ProgramInvocation *inv, + local::ProgramResourceBarrier barrier); + + // Imports a raw hal buffer from an invocation as a storage, attaching any + // needed barriers. + static storage ImportInvocationResultStorage(local::ProgramInvocation *inv, + iree::hal_buffer_ptr buffer); + // The timeline resource holds the back reference to the owning scope, // which keeps all devices alive. Buffers must be destroyed before devices, // so this must be declared first. local::detail::TimelineResource::Ref timeline_resource_; iree::hal_buffer_ptr buffer_; local::ScopedDevice device_; + + friend class shortfin::local::ProgramInvocationMarshalableFactory; + friend class device_array; }; // Wraps an untyped mapping, providing typed access. diff --git a/libshortfin/src/shortfin/array/xtensor_bridge.cc b/libshortfin/src/shortfin/array/xtensor_bridge.cc index 0dc00f9c7..fa36c4ca1 100644 --- a/libshortfin/src/shortfin/array/xtensor_bridge.cc +++ b/libshortfin/src/shortfin/array/xtensor_bridge.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/array/xtensor_bridge.h b/libshortfin/src/shortfin/array/xtensor_bridge.h index a3243e03b..3519f6619 100644 --- a/libshortfin/src/shortfin/array/xtensor_bridge.h +++ b/libshortfin/src/shortfin/array/xtensor_bridge.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/CMakeLists.txt b/libshortfin/src/shortfin/local/CMakeLists.txt index 8ac8dd0d3..37d93bdcb 100644 --- a/libshortfin/src/shortfin/local/CMakeLists.txt +++ b/libshortfin/src/shortfin/local/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: diff --git a/libshortfin/src/shortfin/local/async.cc b/libshortfin/src/shortfin/local/async.cc index a5e005768..0c63ebd70 100644 --- a/libshortfin/src/shortfin/local/async.cc +++ b/libshortfin/src/shortfin/local/async.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -173,7 +173,7 @@ void Future::ThrowFailureWithLockHeld() { if (!state_->done_) { throw std::logic_error("Cannot get result from Future that is not done"); } - SHORTFIN_THROW_IF_ERROR(state_->failure_status_); + SHORTFIN_THROW_IF_ERROR(state_->failure_status_.ConsumeStatus()); } } // namespace shortfin::local diff --git a/libshortfin/src/shortfin/local/async.h b/libshortfin/src/shortfin/local/async.h index 094c90707..75f5545d0 100644 --- a/libshortfin/src/shortfin/local/async.h +++ b/libshortfin/src/shortfin/local/async.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -166,6 +166,16 @@ class SHORTFIN_API TypedFuture : public Future { return *this; } + // Futures are non-nullable, so construct/assign from an rvalue reference + // is just a copy and does not clear the original. + TypedFuture(TypedFuture &&other) : Future(other.state_) { Retain(); } + TypedFuture &operator=(TypedFuture &&other) { + other.Retain(); + Release(); + state_ = other.state_; + return *this; + } + void set_result(ResultTy result) { iree::slim_mutex_lock_guard g(state_->lock_); if (state_->done_) { diff --git a/libshortfin/src/shortfin/local/device.cc b/libshortfin/src/shortfin/local/device.cc index a6b4a9bb2..3cb1bd40d 100644 --- a/libshortfin/src/shortfin/local/device.cc +++ b/libshortfin/src/shortfin/local/device.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/device.h b/libshortfin/src/shortfin/local/device.h index ccfc30ef6..eb24d6a9f 100644 --- a/libshortfin/src/shortfin/local/device.h +++ b/libshortfin/src/shortfin/local/device.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -194,6 +194,7 @@ class SHORTFIN_API DeviceAffinity { return result; } + operator bool() const { return device_ != nullptr; } Device *device() const { return device_; } iree_hal_queue_affinity_t queue_affinity() const { return queue_affinity_; } // Returns the lowest queue ordinal in the affinity set. If there are no diff --git a/libshortfin/src/shortfin/local/messaging.cc b/libshortfin/src/shortfin/local/messaging.cc index 2078f10b9..7e3e166ab 100644 --- a/libshortfin/src/shortfin/local/messaging.cc +++ b/libshortfin/src/shortfin/local/messaging.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/messaging.h b/libshortfin/src/shortfin/local/messaging.h index f006775fe..e60417fd7 100644 --- a/libshortfin/src/shortfin/local/messaging.h +++ b/libshortfin/src/shortfin/local/messaging.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -121,9 +121,9 @@ class SHORTFIN_API Message { // sized field that the allocator can use at it sees fit. Both fields // are managed within a lock_ scope and are optimized for single threaded // access and cross-thread transfers with coarse references. + mutable iree::slim_mutex lock_; mutable intptr_t ref_data_ = 1; mutable detail::MessageRefOwner owner_; - mutable iree::slim_mutex lock_; friend struct detail::MessageRefOwner; }; diff --git a/libshortfin/src/shortfin/local/process.cc b/libshortfin/src/shortfin/local/process.cc index b40b8ce87..3d3f6ff54 100644 --- a/libshortfin/src/shortfin/local/process.cc +++ b/libshortfin/src/shortfin/local/process.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/process.h b/libshortfin/src/shortfin/local/process.h index f1b0b008d..d10c88a3e 100644 --- a/libshortfin/src/shortfin/local/process.h +++ b/libshortfin/src/shortfin/local/process.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/program.cc b/libshortfin/src/shortfin/local/program.cc index cba725096..f3154bd32 100644 --- a/libshortfin/src/shortfin/local/program.cc +++ b/libshortfin/src/shortfin/local/program.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -8,13 +8,85 @@ #include "fmt/core.h" #include "fmt/std.h" +#include "iree/modules/hal/module.h" #include "iree/vm/bytecode/module.h" +#include "shortfin/local/scope.h" #include "shortfin/local/system.h" +#include "shortfin/support/logging.h" namespace shortfin::local { -ProgramModule ProgramModule::Load(System& system, - const std::filesystem::path& path, +namespace { +void GetVmModuleExports(iree_vm_module_t *vm_module, + std::vector &exports) { + auto sig = iree_vm_module_signature(vm_module); + for (iree_host_size_t i = 0; i < sig.export_function_count; ++i) { + iree_vm_function_t f; + SHORTFIN_THROW_IF_ERROR(iree_vm_module_lookup_function_by_ordinal( + vm_module, IREE_VM_FUNCTION_LINKAGE_EXPORT, i, &f)); + exports.emplace_back(to_string_view(iree_vm_function_name(&f))); + } +} +} // namespace + +// -------------------------------------------------------------------------- // +// ProgramFunction +// -------------------------------------------------------------------------- // + +ProgramFunction::ProgramFunction( + std::shared_ptr scope, iree::vm_context_ptr vm_context, + iree_vm_function_t vm_function, + std::optional invocation_model) + : scope_(std::move(scope)), + vm_context_(std::move(vm_context)), + vm_function_(vm_function), + invocation_model_(invocation_model + ? *invocation_model + : GetInvocationModelFromFunction(vm_function)) {} + +ProgramInvocationModel ProgramFunction::GetInvocationModelFromFunction( + iree_vm_function_t &f) { + iree_string_view_t invocation_model_sv = + iree_vm_function_lookup_attr_by_name(&f, IREE_SV("iree.abi.model")); + if (iree_string_view_equal(invocation_model_sv, IREE_SV("coarse-fences"))) { + return ProgramInvocationModel::COARSE_FENCES; + } else if (invocation_model_sv.size == 0) { + return ProgramInvocationModel::NONE; + } else { + logging::warn("Unknown function invocation model '{}': '{}'", + to_string_view(iree_vm_function_name(&f)), + to_string_view(invocation_model_sv)); + return ProgramInvocationModel::UNKNOWN; + } +} + +std::string_view ProgramFunction::name() const { + if (!*this) return {}; + return to_string_view(iree_vm_function_name(&vm_function_)); +} + +std::string_view ProgramFunction::calling_convention() const { + if (!*this) return {}; + return to_string_view( + iree_vm_function_signature(&vm_function_).calling_convention); +} + +ProgramInvocation::Ptr ProgramFunction::CreateInvocation() { + return ProgramInvocation::New(scope_, vm_context_, vm_function_, + invocation_model_); +} + +std::string ProgramFunction::to_s() const { + if (!*this) return std::string("ProgramFunction(NULL)"); + return fmt::format("ProgramFunction({}: {})", name(), calling_convention()); +} + +// -------------------------------------------------------------------------- // +// ProgramModule +// -------------------------------------------------------------------------- // + +ProgramModule ProgramModule::Load(System &system, + const std::filesystem::path &path, bool mmap) { iree::file_contents_ptr contents; iree_file_read_flags_t flags = @@ -53,4 +125,412 @@ std::string ProgramModule::to_s() const { sig.version, fmt::join(exports, ", ")); } +std::vector ProgramModule::exports() const { + std::vector exports; + GetVmModuleExports(vm_module_, exports); + return exports; +} + +// -------------------------------------------------------------------------- // +// Program +// -------------------------------------------------------------------------- // + +Program Program::Load(std::shared_ptr scope, + std::span modules, Options options) { + std::vector all_modules; + std::vector raw_devices; + + // By default, bind all devices in the scope in order to the program. + for (Device *d : scope->raw_devices()) { + raw_devices.push_back(d->hal_device()); + } + + // Add a HAL module. + // TODO: at some point may want to change this to something similar to + // what the tooling does in iree_tooling_resolve_modules - it uses + // iree_vm_module_enumerate_dependencies to walk the dependencies and add the + // required modules only as needed. to start you could use it just to see if + // the hal is used, but as you add other module types for exposing sharkfin + // functionality (or module versions; iree_vm_module_dependency_t has the + // minimum version required so you can switch between them, and whether they + // are optional/required). + auto &system = scope->system(); + iree::vm_module_ptr hal_module; + SHORTFIN_THROW_IF_ERROR( + iree_hal_module_create(system.vm_instance(), raw_devices.size(), + raw_devices.data(), IREE_HAL_MODULE_FLAG_NONE, + system.host_allocator(), hal_module.for_output())); + all_modules.push_back(hal_module); + + // Add explicit modules. + for (auto &pm : modules) { + all_modules.push_back(pm.vm_module()); + } + + // Create the context. + iree::vm_context_ptr context; + iree_vm_context_flags_t flags = IREE_VM_CONTEXT_FLAG_CONCURRENT; + if (options.trace_execution) flags |= IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION; + SHORTFIN_THROW_IF_ERROR(iree_vm_context_create_with_modules( + system.vm_instance(), flags, all_modules.size(), all_modules.data(), + system.host_allocator(), context.for_output())); + + return Program(std::move(scope), std::move(context)); +} + +std::optional Program::LookupFunction(std::string_view name) { + // By convention, we currently name our coarse-fences function variants + // as ending in "$async". These are the ones we want but it is inconvenient. + // Therefore, we probe for that first. + // TODO: We should add attributes to the function that better describe this + // relationship. + iree_vm_function_t f; + if (!name.ends_with("$async")) { + std::string async_name(name); + async_name.append("$async"); + iree_status_t status = iree_vm_context_resolve_function( + vm_context_, to_iree_string_view(async_name), &f); + if (iree_status_is_ok(status)) { + // TODO: Torch import is not setting the coarse-fences abi.model on + // its functions. Get it from there instead of just assuming based on + // name. + return ProgramFunction(scope_, vm_context_, f, + ProgramInvocationModel::COARSE_FENCES); + } else if (!iree_status_is_not_found(status)) { + SHORTFIN_THROW_IF_ERROR(status); + } + } + + // Resolve the exactly named function. + iree_status_t status = iree_vm_context_resolve_function( + vm_context_, to_iree_string_view(name), &f); + if (iree_status_is_not_found(status)) return {}; + SHORTFIN_THROW_IF_ERROR(status); + return ProgramFunction(scope_, vm_context_, f); +} + +ProgramFunction Program::LookupRequiredFunction(std::string_view name) { + auto f = LookupFunction(name); + if (!f) { + throw std::invalid_argument( + fmt::format("Function '{}' not found in program. Available exports: {}", + name, fmt::join(exports(), ", "))); + } + return std::move(*f); +} + +std::vector Program::exports() const { + std::vector results; + + // Iterate in reverse since "user modules" are typically last. + int module_count = iree_vm_context_module_count(vm_context_); + for (int i = module_count - 1; i >= 0; --i) { + auto vm_module = iree_vm_context_module_at(vm_context_, i); + std::string_view module_name = + to_string_view(iree_vm_module_name(vm_module)); + std::vector names; + GetVmModuleExports(vm_module, names); + for (auto &name : names) { + results.push_back(fmt::format("{}.{}", module_name, name)); + } + } + return results; +} + +// -------------------------------------------------------------------------- // +// ProgramInvocation +// -------------------------------------------------------------------------- // + +iree_vm_list_t *ProgramInvocation::arg_list() { + // The arg list is located immediately after this, allocated as a trailing + // data structure. + return reinterpret_cast(reinterpret_cast(this) + + sizeof(*this)); +} + +void ProgramInvocation::Deleter::operator()(ProgramInvocation *inst) { + inst->~ProgramInvocation(); + uint8_t *memory = static_cast(static_cast(inst)); + + // Trailing arg list and result list. The arg list pointer is only available + // at construction, so we use the knowledge that it is stored right after + // the object. The result_list_ is available for the life of the invocation. + iree_vm_list_deinitialize(static_cast( + static_cast(memory + sizeof(ProgramInvocation)))); + iree_vm_list_deinitialize(inst->result_list_); + + // Was allocated in New as a uint8_t[] so delete it by whence it came. + delete[] memory; +} + +ProgramInvocation::ProgramInvocation() = default; +ProgramInvocation::~ProgramInvocation() { + if (!scheduled()) { + // This instance was dropped on the floor before scheduling. + // Clean up the initialization parameters. + iree::vm_context_ptr drop = + iree::vm_context_ptr::steal_reference(state.params.context); + } +} + +ProgramInvocation::Ptr ProgramInvocation::New( + std::shared_ptr scope, iree::vm_context_ptr vm_context, + iree_vm_function_t &vm_function, ProgramInvocationModel invocation_model) { + auto sig = iree_vm_function_signature(&vm_function); + iree_host_size_t arg_count; + iree_host_size_t result_count; + SHORTFIN_THROW_IF_ERROR(iree_vm_function_call_count_arguments_and_results( + &sig, &arg_count, &result_count)); + + // Compute size of trailing arg/result storage. + auto variant_type_def = iree_vm_make_undefined_type_def(); + iree_host_size_t arg_storage_size = + iree_vm_list_storage_size(&variant_type_def, arg_count); + iree_host_size_t result_storage_size = + iree_vm_list_storage_size(&variant_type_def, result_count); + + // Allocate storage for the ProgramInvocation, arg, result list and placement + // new the ProgramInvocation into the storage area. + std::unique_ptr inst_storage( + new uint8_t[sizeof(ProgramInvocation) + arg_storage_size + + result_storage_size]); + new (inst_storage.get()) ProgramInvocation(); + + // Initialize trailing lists. Abort on failure since this is a bug and we + // would otherwise leak. + iree_vm_list_t *arg_list; + iree_vm_list_t *result_list; + IREE_CHECK_OK(iree_vm_list_initialize( + {.data = inst_storage.get() + sizeof(ProgramInvocation), + .data_length = arg_storage_size}, + &variant_type_def, arg_count, &arg_list)); + IREE_CHECK_OK(iree_vm_list_initialize( + {.data = + inst_storage.get() + sizeof(ProgramInvocation) + arg_storage_size, + .data_length = result_storage_size}, + &variant_type_def, result_count, &result_list)); + + Ptr inst(static_cast( + static_cast(inst_storage.release())), + Deleter()); + inst->scope_ = std::move(scope); + inst->state.params.context = + vm_context.release(); // Ref transfer to ProgramInvocation. + inst->state.params.function = vm_function; + inst->state.params.invocation_model = invocation_model; + inst->result_list_ = result_list; + return inst; +} + +void ProgramInvocation::CheckNotScheduled() { + if (scheduled()) { + throw std::logic_error("Cannot mutate an invocation once scheduled."); + } +} + +void ProgramInvocation::AddArg(iree::vm_opaque_ref ref) { + CheckNotScheduled(); + SHORTFIN_THROW_IF_ERROR(iree_vm_list_push_ref_move(arg_list(), &ref)); +} + +void ProgramInvocation::AddArg(iree_vm_ref_t *ref) { + CheckNotScheduled(); + SHORTFIN_THROW_IF_ERROR(iree_vm_list_push_ref_retain(arg_list(), ref)); +} + +iree_status_t ProgramInvocation::FinalizeCallingConvention( + iree_vm_list_t *arg_list, iree_vm_function_t &function, + ProgramInvocationModel invocation_model) { + // Handle post-processing invocation model setup. + if (invocation_model == ProgramInvocationModel::COARSE_FENCES) { + // If we have a device_selection, set up to signal the leader account. + if (device_selection_) { + ScopedDevice scoped_device(*scope(), device_selection_); + auto &sched_account = + scope()->scheduler().GetDefaultAccount(scoped_device); + iree_hal_fence_t *wait_fence = this->wait_fence(); + iree_hal_semaphore_t *timeline_sem = sched_account.timeline_sem(); + uint64_t timeline_now = sched_account.timeline_idle_timepoint(); + SHORTFIN_SCHED_LOG("Invocation {}: Wait on account timeline {}@{}", + static_cast(this), + static_cast(timeline_sem), timeline_now); + IREE_RETURN_IF_ERROR( + iree_hal_fence_insert(wait_fence, timeline_sem, timeline_now)); + signal_sem_ = sched_account.timeline_sem(); + signal_timepoint_ = sched_account.timeline_acquire_timepoint(); + } + + // Push wait fence (or null if no wait needed). + ::iree::vm::ref wait_ref; + if (wait_fence_) { + ::iree::vm::retain_ref(wait_fence()); + } + IREE_RETURN_IF_ERROR(iree_vm_list_push_ref_move(arg_list, wait_ref)); + + // Create and push signal fence (or null if no signal needed). + ::iree::vm::ref signal_ref; + if (signal_sem_) { + SHORTFIN_SCHED_LOG("Invocation {}: Set signal {}@{}", + static_cast(this), + static_cast(signal_sem_), signal_timepoint_); + IREE_RETURN_IF_ERROR( + iree_hal_fence_create_at(signal_sem_, signal_timepoint_, + scope()->host_allocator(), &signal_ref)); + } + IREE_RETURN_IF_ERROR(iree_vm_list_push_ref_move(arg_list, signal_ref)); + } else { + logging::warn( + "Invoking function '{}' with unknown or synchronous invocation model " + "is not fully supported", + to_string_view(iree_vm_function_name(&function))); + } + + return iree_ok_status(); +} + +ProgramInvocation::Future ProgramInvocation::Invoke( + ProgramInvocation::Ptr invocation) { + invocation->CheckNotScheduled(); + + Worker &worker = invocation->scope_->worker(); + // We're about to overwrite the instance level storage for params, so move + // it to the stack and access there. + Params params = invocation->state.params; + + auto schedule = [](ProgramInvocation *raw_invocation, Worker *worker, + iree_vm_context_t *owned_context, + iree_vm_function_t function, + ProgramInvocationModel invocation_model, + std::optional failure_future) { + auto complete_callback = + [](void *user_data, iree_loop_t loop, iree_status_t status, + iree_vm_list_t *outputs) noexcept -> iree_status_t { + // Async invocation helpfully gives us a retained reference to the + // outputs, but we already have one statically on the + // ProgramInvocation. So release this one, which makes it safe to + // deallocate the ProgramInvocation at any point after this (there + // must be no live references to inputs/outputs when the + // ProgramInvocation::Ptr deleter is invoked). + iree::vm_list_ptr::steal_reference(outputs); + + // Repatriate the ProgramInvocation. + ProgramInvocation::Ptr invocation( + static_cast(user_data)); + ProgramInvocation *raw_invocation = invocation.get(); + if (iree_status_is_ok(status)) { + raw_invocation->future_->set_result(std::move(invocation)); + } else { + raw_invocation->future_->set_failure(status); + } + + // Must release the future from the invocation to break the + // circular reference (we are setting the invocation as the result + // of the future). + raw_invocation->future_.reset(); + + return iree_ok_status(); + }; + + ProgramInvocation::Ptr invocation(raw_invocation); + iree_status_t status = iree_ok_status(); + + // Multiple steps needed to schedule need to all exit via the same + // path. + if (iree_status_is_ok(status)) { + status = invocation->scope()->scheduler().FlushWithStatus(); + } + if (iree_status_is_ok(status)) { + status = invocation->FinalizeCallingConvention( + invocation->arg_list(), function, invocation_model); + } + if (iree_status_is_ok(status)) { + status = iree_vm_async_invoke(worker->loop(), + &invocation->state.async_invoke_state, + owned_context, function, + /*flags=*/IREE_VM_INVOCATION_FLAG_NONE, + /*policy=*/nullptr, + /*inputs=*/invocation->arg_list(), + /*outputs=*/invocation->result_list_, + iree_allocator_system(), +complete_callback, + /*user_data=*/invocation.get()); + } + + // Regardless of status, the context reference we were holding is no + // longer needed. Drop it on the floor. + iree::vm_context_ptr::steal_reference(owned_context); + + // On success, then the complete callback takes ownership of the + // invocation, so we release it here and return. We have to treat + // the invocation as possibly deallocated at this point, since the + // async invocation may have finished already. + if (iree_status_is_ok(status)) { + invocation.release(); + } else if (failure_future) { + // Requested to set any failure on the future. + failure_future->set_failure(status); + } else { + // Synchronous: just throw. + SHORTFIN_THROW_IF_ERROR(status); + } + }; + + // Transition to the scheduled state. + invocation->future_.emplace(&worker); + auto fork_future = *invocation->future_; + invocation->scheduled_ = true; + + if (&worker == Worker::GetCurrent()) { + // On the same worker: fast-path directly to the loop. + schedule(invocation.release(), &worker, params.context, params.function, + params.invocation_model, /*failure_future=*/{}); + } else { + // Cross worker coordination: submit an external task to bootstrap. + auto bound_schedule = + std::bind(schedule, invocation.release(), &worker, params.context, + params.function, params.invocation_model, + /*failure_future=*/fork_future); + worker.CallThreadsafe(bound_schedule); + } + + return fork_future; +} + +iree_host_size_t ProgramInvocation::results_size() { + return iree_vm_list_size(result_list_); +} + +iree::vm_opaque_ref ProgramInvocation::result_ref(iree_host_size_t i) { + iree::vm_opaque_ref out_value; + auto status = iree_vm_list_get_ref_retain(result_list_, i, &out_value); + if (iree_status_is_failed_precondition(status)) return {}; + SHORTFIN_THROW_IF_ERROR(status, "accessing invocation result"); + return out_value; +} + +iree_hal_fence_t *ProgramInvocation::wait_fence() { + if (!wait_fence_) { + wait_fence_ = scope_->scheduler().NewFence(); + } + return wait_fence_.get(); +} + +void ProgramInvocation::wait_insert(iree_hal_semaphore_list_t sem_list) { + iree_hal_fence_t *f = wait_fence(); + for (iree_host_size_t i = 0; i < sem_list.count; ++i) { + SHORTFIN_SCHED_LOG("Invocation {}: Wait on {}@{}", + static_cast(this), + static_cast(sem_list.semaphores[i]), + sem_list.payload_values[i]); + SHORTFIN_THROW_IF_ERROR(iree_hal_fence_insert(f, sem_list.semaphores[i], + sem_list.payload_values[i])); + } +} + +void ProgramInvocation::DeviceSelect(DeviceAffinity device_affinity) { + CheckNotScheduled(); + SHORTFIN_SCHED_LOG("Invocation {}: DeviceSelect {}", + static_cast(this), device_affinity.to_s()); + device_selection_ |= device_affinity; +} + } // namespace shortfin::local diff --git a/libshortfin/src/shortfin/local/program.h b/libshortfin/src/shortfin/local/program.h index 40637a768..d3c2984e7 100644 --- a/libshortfin/src/shortfin/local/program.h +++ b/libshortfin/src/shortfin/local/program.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -8,15 +8,205 @@ #define SHORTFIN_LOCAL_PROGRAM_H #include +#include +#include #include +#include +#include "shortfin/local/async.h" +#include "shortfin/local/device.h" +#include "shortfin/local/program_interfaces.h" +#include "shortfin/local/worker.h" #include "shortfin/support/api.h" #include "shortfin/support/iree_helpers.h" namespace shortfin::local { +class SHORTFIN_API Scope; class SHORTFIN_API System; +enum class ProgramInvocationModel { + // Uses the coarse-fences invocation model. In this model, the last two + // arguments are a wait and signal fence, which are used for function-level + // scheduling. + COARSE_FENCES, + // The function was not annotated with an invocation model. + NONE, + // The function is not annotated or is simple/synchronous. + UNKNOWN, +}; + +// State related to making an invocation of a function on a program. +// +// Since ownership of this object is transferred to the loop/callback and +// internal pointers into it must remain stable, it is only valid to heap +// allocate it. +class SHORTFIN_API ProgramInvocation { + struct Deleter { + void operator()(ProgramInvocation *); + }; + + public: + // The fact that we traffic in invocation pointers based on unique_ptr + // is incidental. By cloaking its public interface this way, we use the + // unique_ptr machinery but template meta-programming that is specialized + // for unique_ptr sees this as a bespoke class (which is what we want because + // ownership semantics are special). + class Ptr : private std::unique_ptr { + public: + using unique_ptr::unique_ptr; + using unique_ptr::operator=; + using unique_ptr::operator->; + using unique_ptr::operator bool; + using unique_ptr::get; + using unique_ptr::release; + }; + static_assert(sizeof(Ptr) == sizeof(void *)); + using Future = TypedFuture; + + static Ptr New(std::shared_ptr scope, iree::vm_context_ptr vm_context, + iree_vm_function_t &vm_function, + ProgramInvocationModel invocation_model); + ProgramInvocation(const ProgramInvocation &) = delete; + ProgramInvocation &operator=(const ProgramInvocation &) = delete; + ProgramInvocation &operator=(ProgramInvocation &&) = delete; + ProgramInvocation(ProgramInvocation &&inv) = delete; + ~ProgramInvocation(); + + // Whether the ProgramInvocation has entered the scheduled state. Once + // scheduled, arguments and initialization parameters can no longer be + // accessed. + bool scheduled() const { return scheduled_; } + + // The scope this invocation was scheduled against. + Scope *scope() const { return scope_.get(); } + + // Adds wait barriers to the invocation. For coarse fences invocations, these + // will cause execution of the function to wait until all sempahores added + // thusly are satisfied. + void wait_insert(iree_hal_semaphore_list_t sem_list); + + // Adds a marshalable argument with a configurable concurrency barrier. + void AddArg(ProgramInvocationMarshalable &marshalable, + ProgramResourceBarrier barrier = ProgramResourceBarrier::READ); + + // Adds a ref object argument. This low level interface directly adds a + // reference object and does not manipulate any execution barriers. + void AddArg(iree::vm_opaque_ref ref); // Moves a reference in. + void AddArg(iree_vm_ref_t *ref); // Borrows the reference. + + // Transfers ownership of an invocation and schedules it on worker, returning + // a future that will resolve to the owned invocation upon completion. + static ProgramInvocation::Future Invoke(ProgramInvocation::Ptr invocation); + + // Gets the number of outputs. + iree_host_size_t results_size(); + + // Gets the i'th result as an opaque ref object. Returns a null ref if the + // result is a primitive. Outputs accessed in this way are not marshaled + // nor do they have concurrency barriers applied. + iree::vm_opaque_ref result_ref(iree_host_size_t i); + + // As arguments are processed, the device they are associated with should be + // passed here. The accumulation of these will drive the selection of the + // scheduling account used for the invocation timeline. In the absence of + // a specific directive, all arguments implicated in scheduling (i.e. + // excepting those with ProgramResourceBarrier::NONE) must be on the same + // logical device and only differ by queue affinity. + // This method will raise an exception if the implied semantics are violated. + void DeviceSelect(DeviceAffinity device_affinity); + + // Selected device affinity used for scheduling. + const DeviceAffinity &device_selection() { return device_selection_; } + + // If this invocation provides coarse signaling of result availability, + // the semaphore and timepoint are returned here. If the semaphore is null, + // then coarse signaling is not available. + // Valid after invocation has been scheduled. + std::pair coarse_signal() { + return std::make_pair(signal_sem_, signal_timepoint_); + } + + private: + ProgramInvocation(); + void CheckNotScheduled(); + + // Returns a pointer to the trailing arg list. + iree_vm_list_t *arg_list(); + + // Accesses the invocation owned wait fence, creating it if needed. + iree_hal_fence_t *wait_fence(); + + // Called as part of scheduling to finalize the calling convention and + // invocation model after user arguments have been added. Because this is + // potentially run in a foreign callback context, it uses iree_status_t + // error reporting vs exceptions. + iree_status_t FinalizeCallingConvention( + iree_vm_list_t *arg_list, iree_vm_function_t &function, + ProgramInvocationModel invocation_model); + + // Parameters needed to make the async call are stored at construction time + // up until the point the call is made in the params union. When invoking, + // these will be copied to the stack and passed to the async invocation, + // which initializes the async_invoke_state. Phasing it like this saves + // memory that would otherwise be retained for the life of the invocation. + // This must not contain entities that require destruction or cannot be + // trivially copied. + struct Params { + // Context is retained upon construction and released when scheduled. + iree_vm_context_t *context; + iree_vm_function_t function; + ProgramInvocationModel invocation_model; + }; + union State { + State() { new (¶ms) Params(); } + ~State() {} + Params params; + iree_vm_async_invoke_state_t async_invoke_state; + } state; + + std::shared_ptr scope_; + iree_vm_list_t *result_list_ = nullptr; + std::optional future_; + iree::hal_fence_ptr wait_fence_; + iree_hal_semaphore_t *signal_sem_ = nullptr; + uint64_t signal_timepoint_ = 0; + DeviceAffinity device_selection_; + bool scheduled_ = false; +}; + +// References a function in a Program. +class SHORTFIN_API ProgramFunction { + public: + operator bool() const { return vm_context_; } + + std::string_view name() const; + std::string_view calling_convention() const; + ProgramInvocationModel invocation_model() const { return invocation_model_; } + + ProgramInvocation::Ptr CreateInvocation(); + + std::string to_s() const; + + operator iree_vm_context_t *() { return vm_context_.get(); } + operator iree_vm_function_t &() { return vm_function_; } + + private: + ProgramFunction(std::shared_ptr scope, iree::vm_context_ptr vm_context, + iree_vm_function_t vm_function, + std::optional invocation_model = {}); + + static ProgramInvocationModel GetInvocationModelFromFunction( + iree_vm_function_t &f); + + // The context that this function was resolved against. + std::shared_ptr scope_; + iree::vm_context_ptr vm_context_; + iree_vm_function_t vm_function_; + ProgramInvocationModel invocation_model_; + friend class Program; +}; + // High level API for working with program modules. Think of a module as // a shared library in a traditional Unix system: // @@ -36,13 +226,16 @@ class SHORTFIN_API System; class SHORTFIN_API ProgramModule { public: std::string to_s() const; - iree_vm_module_t* vm_module() const { return vm_module_; } + iree_vm_module_t *vm_module() const { return vm_module_; } std::string_view name() const; // Loads a dynamic bytecode module (VMFB) from a path on the file system. - static ProgramModule Load(System& system, const std::filesystem::path& path, + static ProgramModule Load(System &system, const std::filesystem::path &path, bool mmap = true); + // Gets the name of all exported functions. + std::vector exports() const; + protected: explicit ProgramModule(iree::vm_module_ptr vm_module) : vm_module_(std::move(vm_module)) {} @@ -52,28 +245,48 @@ class SHORTFIN_API ProgramModule { }; // Programs consist of ProgramModules instantiated together and capable of -// having functions invoked on them. While it is possible to construct -// programs that do not depend on device-associated state, the dominant -// use case is for programs that are compiled to operate against the device -// HAL with a list of concrete devices. Such programs are constructed from -// a Scope. +// having functions invoked on them. While the underlying programming model +// is a bit broader and can be exploited in various advanced way, generally, +// a program should be thought of as a fiber, and it is therefore bound to +// a Scope, which provides a logical thread of execution. By default, all +// invocations will take place in logical order (there are certain ways to +// violate this constraint safely that are provided for separately). // -// While the concurrency model for programs is technically a bit broader, the -// intended use is for them to be interacted with on a single Worker in a -// non-blocking fashion. There are many advanced ways that programs can be -// constructed to straddle devices, scopes, and workers, but that is left as -// an advanced use case. +// The program will source any needed parameters from the System and it will +// make an effort to cache them for proper locality on individual devices +// (TODO: make this actually true). class SHORTFIN_API Program { public: struct Options { + Options() {} + // Enables program-wide execution tracing (to stderr). bool trace_execution = false; }; + // Loads a program attached to a scope with a list of user provided modules + // and options. + static Program Load(std::shared_ptr scope, + std::span modules, + Options options = {}); + + // Looks up a public function by fully qualified name (i.e. module.function). + // Returns nothing if not found. + std::optional LookupFunction(std::string_view name); + + // Looks up a public function by fully qualified name, throwing an + // invalid_argument exception on failure to find. + ProgramFunction LookupRequiredFunction(std::string_view name); + + // Gets the name of all exported functions. + std::vector exports() const; + private: - explicit Program(iree::vm_context_ptr context) - : context_(std::move(context)) {} - iree::vm_context_ptr context_; + explicit Program(std::shared_ptr scope, + iree::vm_context_ptr vm_context) + : scope_(std::move(scope)), vm_context_(std::move(vm_context)) {} + std::shared_ptr scope_; + iree::vm_context_ptr vm_context_; friend class Scope; }; diff --git a/libshortfin/src/shortfin/local/program_interfaces.h b/libshortfin/src/shortfin/local/program_interfaces.h new file mode 100644 index 000000000..b376c75d3 --- /dev/null +++ b/libshortfin/src/shortfin/local/program_interfaces.h @@ -0,0 +1,85 @@ +// Copyright 2024 Advanced Micro Devices, Inc. +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Standalone interfaces needed for marshaling as part of a ProgramInvocation.h. +// They are available in this dep-free header in order to ease the burden on +// types that would otherwise need to pull in all of the includes. + +#ifndef SHORTFIN_LOCAL_PROGRAM_INTERFACES_H +#define SHORTFIN_LOCAL_PROGRAM_INTERFACES_H + +#include "shortfin/support/api.h" +#include "shortfin/support/iree_helpers.h" + +namespace shortfin::local { + +class SHORTFIN_API ProgramInvocation; + +// The type of barrier that should be managed for a program resource. +enum class ProgramResourceBarrier { + // The caller has explicitly not stated a preference. + DEFAULT, + + // The argument will be used by the program for input and the program + // must not perform operations on it until all pending mutations have + // been completed. Concurrent reads/uses are permitted. + // This is the default concurrency in most situations. + READ, + + // The argument will be used for input/output and the program must not + // perform operations on it until all prior mutations and uses have been + // complete. + WRITE, + + // No concurrency barriers will be emplaced on behalf of the argument, + // explicitly allowing racy access. The program and the caller must + // ensure that only valid accesses are made. + NONE, +}; + +// Implemented by a class if it can marshal itself to an invocation as an +// argument. +class SHORTFIN_API ProgramInvocationMarshalable { + public: + // Adds this object as an invocation argument. + virtual void AddAsInvocationArgument(ProgramInvocation *inv, + ProgramResourceBarrier barrier) = 0; +}; + +// Trampoline class that has visibility into marshalable types and can be used +// to construct them from an invocation reference. +class SHORTFIN_API ProgramInvocationMarshalableFactory { + public: + // Instantiates a new `T` from an opaque reference retrieved from an + // invocation result. This will call through to a factory on the type to + // construct a new user-value and setup any needed barriers from the + // invocation. + // + // In order for a type to be eligible for such usage, it must expose a + // `T CreateFromInvocationResultRef(ProgramInvocation *inv, + // iree::vm_opaque_ref)` static method. The type `T` must be friends with this + // class. + template + static T CreateFromInvocationResultRef(ProgramInvocation *inv, + iree::vm_opaque_ref ref) { + return T::CreateFromInvocationResultRef(inv, std::move(ref)); + } + + // Gets the type id that corresponds to this marshalable type. + // + // Marshalable types should define the same method. + // It is recommended that these type methods are defined in shortfin + // implementation files (not headers) since that ensures that no cross-DSO + // symbol visibility issues can transpire. + template + static iree_vm_ref_type_t invocation_marshalable_type() { + return T::invocation_marshalable_type(); + } +}; + +} // namespace shortfin::local + +#endif // SHORTFIN_LOCAL_PROGRAM_INTERFACES_H diff --git a/libshortfin/src/shortfin/local/scheduler.cc b/libshortfin/src/shortfin/local/scheduler.cc index c5a9fc062..6f5581270 100644 --- a/libshortfin/src/shortfin/local/scheduler.cc +++ b/libshortfin/src/shortfin/local/scheduler.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -12,6 +12,26 @@ namespace shortfin::local::detail { +namespace { + +[[maybe_unused]] std::string SummarizeFence(iree_hal_fence_t *fence) { + if (!SHORTFIN_SCHED_LOG_ENABLED) { + return std::string(); + } + std::string result("fence("); + iree_hal_semaphore_list_t list = iree_hal_fence_semaphore_list(fence); + for (iree_host_size_t i = 0; i < list.count; ++i) { + if (i > 0) result.append(", "); + result.append(fmt::format("[{}@{}]", + static_cast(list.semaphores[i]), + list.payload_values[i])); + } + result.append(")"); + return result; +} + +} // namespace + // -------------------------------------------------------------------------- // // Account // -------------------------------------------------------------------------- // @@ -30,9 +50,6 @@ void Account::Initialize() { void Account::Reset() { active_tx_type_ = TransactionType::NONE; - // if (active_command_buffer_) { - // iree_hal_command_buffer_end(active_command_buffer_); - // } active_command_buffer_.reset(); } @@ -55,11 +72,15 @@ CompletionEvent Account::OnSync() { iree::shared_event::ref satisfied(false); iree::hal_semaphore_ptr sem = sem_; auto idle_timepoint = idle_timepoint_; + SHORTFIN_SCHED_LOG("OnSync::Wait({}@{})", static_cast(sem.get()), + idle_timepoint); scheduler_.system().blocking_executor().Schedule( [sem = std::move(sem), idle_timepoint, satisfied]() { iree_status_t status = iree_hal_semaphore_wait( sem, idle_timepoint, iree_infinite_timeout()); IREE_CHECK_OK(status); + SHORTFIN_SCHED_LOG("OnSync::Complete({}@{})", + static_cast(sem.get()), idle_timepoint); satisfied->set(); }); return CompletionEvent(satisfied); @@ -89,6 +110,10 @@ void TimelineResource::use_barrier_insert(iree_hal_semaphore_t *sem, iree_hal_fence_insert(use_barrier_fence_, sem, timepoint)); } +iree_allocator_t TimelineResource::host_allocator() { + return scope_->host_allocator(); +} + // -------------------------------------------------------------------------- // // Scheduler // -------------------------------------------------------------------------- // @@ -140,8 +165,10 @@ void Scheduler::AppendCommandBuffer(ScopedDevice &device, TransactionType tx_type, std::function callback) { Account &account = GetDefaultAccount(device); - auto needed_affinity_bits = device.affinity().queue_affinity(); + SHORTFIN_SCHED_LOG( + "AppendCommandBuffer(account=0x{:x}, tx_type={}, queue_affinity={}):", + account.id(), static_cast(tx_type), needed_affinity_bits); // Initialize a fresh command buffer if needed. if (!account.active_command_buffer_) { @@ -181,6 +208,11 @@ void Scheduler::AppendCommandBuffer(ScopedDevice &device, account.active_deps_ = std::move(new_active_deps); account.active_command_buffer_ = std::move(new_cb); account.idle_timepoint_ += 1; + SHORTFIN_SCHED_LOG( + " : New command buffer (category={}, idle_timepoint={})", category, + account.idle_timepoint_); + } else { + SHORTFIN_SCHED_LOG(" : Continue active command buffer"); } // Perform the mutation. @@ -192,21 +224,29 @@ void Scheduler::AppendCommandBuffer(ScopedDevice &device, } } -void Scheduler::Flush() { +iree_status_t Scheduler::FlushWithStatus() noexcept { // This loop is optimized for a small number of accounts, where it is // fine to just linearly probe. If this ever becomes cumbersome, we can // maintain a dirty list which is appended to when an account transitions // from idle to active. for (Account &account : accounts_) { if (!account.active_command_buffer_) continue; - iree_hal_semaphore_t *signal_sem = account.sem_; uint64_t signal_timepoint = account.idle_timepoint_; iree_hal_command_buffer_t *active_command_buffer = account.active_command_buffer_; iree_hal_buffer_binding_table_t binding_tables = iree_hal_buffer_binding_table_empty(); - SHORTFIN_THROW_IF_ERROR(iree_hal_device_queue_execute( + + SHORTFIN_SCHED_LOG( + "Flush command buffer (account=0x{:x}, queue_affinity={}, " + "signal_timepoint={}, deps={})", + account.id(), account.active_queue_affinity_bits_, signal_timepoint, + SummarizeFence(account.active_deps_)); + + // End recording and submit. + IREE_RETURN_IF_ERROR(iree_hal_command_buffer_end(active_command_buffer)); + IREE_RETURN_IF_ERROR(iree_hal_device_queue_execute( account.hal_device(), /*queue_affinity=*/account.active_queue_affinity_bits_, /*wait_sempahore_list=*/account.active_deps_ @@ -223,6 +263,14 @@ void Scheduler::Flush() { /*binding_tables=*/&binding_tables)); account.Reset(); } + return iree_ok_status(); +} + +iree::hal_fence_ptr Scheduler::NewFence() { + iree::hal_fence_ptr fence; + iree_hal_fence_create(semaphore_count_, system_.host_allocator(), + fence.for_output()); + return fence; } } // namespace shortfin::local::detail diff --git a/libshortfin/src/shortfin/local/scheduler.h b/libshortfin/src/shortfin/local/scheduler.h index 2f606ced3..5c514b1bb 100644 --- a/libshortfin/src/shortfin/local/scheduler.h +++ b/libshortfin/src/shortfin/local/scheduler.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -142,6 +142,8 @@ class SHORTFIN_API TimelineResource { return iree_hal_fence_semaphore_list(use_barrier_fence_); } + iree_allocator_t host_allocator(); + private: TimelineResource(std::shared_ptr scope, size_t semaphore_capacity); ~TimelineResource(); @@ -174,7 +176,11 @@ class SHORTFIN_API Account { Account(Scheduler &scheduler, Device *device); Device *device() const { return device_; } iree_hal_device_t *hal_device() { return hal_device_; } + size_t semaphore_count() const { return 1; } + // Gets a unique integer id for this account. Currently just the address of + // the sem, but can be derived from any owned entity. + uintptr_t id() const { return reinterpret_cast(sem_.get()); } // Accesses the active command buffer. This will only be non-null if a // pending transaction has been set up (i.e. via AppendCommandBuffer). @@ -188,6 +194,7 @@ class SHORTFIN_API Account { // Queue timeline. iree_hal_semaphore_t *timeline_sem() { return sem_; } uint64_t timeline_idle_timepoint() { return idle_timepoint_; } + uint64_t timeline_acquire_timepoint() { return ++idle_timepoint_; } // Returns a future that is satisfied when the timeline of this account // reaches its current idle timepoint (i.e. all currently pending work @@ -248,7 +255,8 @@ class SHORTFIN_API Scheduler { std::function callback); // Flushes any pending accounts that have accumulated commands. - void Flush(); + iree_status_t FlushWithStatus() noexcept; + void Flush() { SHORTFIN_THROW_IF_ERROR(FlushWithStatus()); } // Gets a fresh TimelineResource which can be used for tracking resource // read/write and setting barriers. Note that these are all allocated fresh @@ -258,6 +266,10 @@ class SHORTFIN_API Scheduler { new TimelineResource(std::move(scope), semaphore_count_)); } + // Creates a new fence with capacity for all semaphores that are extant at + // the point of the call. + iree::hal_fence_ptr NewFence(); + System &system() { return system_; } private: diff --git a/libshortfin/src/shortfin/local/scope.cc b/libshortfin/src/shortfin/local/scope.cc index 39784f196..e92368a52 100644 --- a/libshortfin/src/shortfin/local/scope.cc +++ b/libshortfin/src/shortfin/local/scope.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,7 +9,6 @@ #include #include -#include "iree/modules/hal/module.h" #include "shortfin/local/system.h" #include "shortfin/support/logging.h" @@ -90,48 +89,6 @@ std::vector Scope::device_names() const { return names; } -Program Scope::LoadUnboundProgram(std::span modules, - Program::Options options) { - std::vector all_modules; - std::vector raw_devices; - - // By default, bind all devices in the scope in order to the program. - for (Device *d : devices_) { - raw_devices.push_back(d->hal_device()); - } - - // Add a HAL module. - // TODO: at some point may want to change this to something similar to - // what the tooling does in iree_tooling_resolve_modules - it uses - // iree_vm_module_enumerate_dependencies to walk the dependencies and add the - // required modules only as needed. to start you could use it just to see if - // the hal is used, but as you add other module types for exposing sharkfin - // functionality (or module versions; iree_vm_module_dependency_t has the - // minimum version required so you can switch between them, and whether they - // are optional/required). - iree::vm_module_ptr hal_module; - SHORTFIN_THROW_IF_ERROR(iree_hal_module_create( - system().vm_instance(), raw_devices.size(), raw_devices.data(), - IREE_HAL_MODULE_FLAG_NONE, system().host_allocator(), - hal_module.for_output())); - all_modules.push_back(hal_module); - - // Add explicit modules. - for (auto &pm : modules) { - all_modules.push_back(pm.vm_module()); - } - - // Create the context. - iree::vm_context_ptr context; - iree_vm_context_flags_t flags = IREE_VM_CONTEXT_FLAG_CONCURRENT; - if (options.trace_execution) flags |= IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION; - SHORTFIN_THROW_IF_ERROR(iree_vm_context_create_with_modules( - system().vm_instance(), flags, all_modules.size(), all_modules.data(), - system().host_allocator(), context.for_output())); - - return Program(std::move(context)); -} - // -------------------------------------------------------------------------- // // ScopedDevice // -------------------------------------------------------------------------- // diff --git a/libshortfin/src/shortfin/local/scope.h b/libshortfin/src/shortfin/local/scope.h index 0cb566b89..14c4d3749 100644 --- a/libshortfin/src/shortfin/local/scope.h +++ b/libshortfin/src/shortfin/local/scope.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -132,14 +132,6 @@ class SHORTFIN_API Scope : public std::enable_shared_from_this { return scheduler().NewTimelineResource(shared_ptr()); } - // Loads a program from a list of modules onto the devices managed by this - // scope. The resulting program is not bound to this scope and can be imported - // into compatible scopes for actual execution. - // TODO: This is temporary during API evolution: a higher level API that - // includes all module concepts, params, etc is needed. - Program LoadUnboundProgram(std::span modules, - Program::Options options = {}); - private: void AddDevice(std::string_view device_class, Device *device); void Initialize(); // Called after all devices are added. diff --git a/libshortfin/src/shortfin/local/system.cc b/libshortfin/src/shortfin/local/system.cc index 2eaf3eaf7..23ecbc088 100644 --- a/libshortfin/src/shortfin/local/system.cc +++ b/libshortfin/src/shortfin/local/system.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/system.h b/libshortfin/src/shortfin/local/system.h index cb5c70808..82fd4b489 100644 --- a/libshortfin/src/shortfin/local/system.h +++ b/libshortfin/src/shortfin/local/system.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/systems/CMakeLists.txt b/libshortfin/src/shortfin/local/systems/CMakeLists.txt index f2c8cef24..effd15204 100644 --- a/libshortfin/src/shortfin/local/systems/CMakeLists.txt +++ b/libshortfin/src/shortfin/local/systems/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: diff --git a/libshortfin/src/shortfin/local/systems/amdgpu.cc b/libshortfin/src/shortfin/local/systems/amdgpu.cc index fbbbe5d9b..b12cb1a5f 100644 --- a/libshortfin/src/shortfin/local/systems/amdgpu.cc +++ b/libshortfin/src/shortfin/local/systems/amdgpu.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/systems/amdgpu.h b/libshortfin/src/shortfin/local/systems/amdgpu.h index 4d0fa5c58..b1a182a8d 100644 --- a/libshortfin/src/shortfin/local/systems/amdgpu.h +++ b/libshortfin/src/shortfin/local/systems/amdgpu.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/systems/host.cc b/libshortfin/src/shortfin/local/systems/host.cc index 2f57df284..311c13398 100644 --- a/libshortfin/src/shortfin/local/systems/host.cc +++ b/libshortfin/src/shortfin/local/systems/host.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/systems/host.h b/libshortfin/src/shortfin/local/systems/host.h index 7827157bb..0655748ab 100644 --- a/libshortfin/src/shortfin/local/systems/host.h +++ b/libshortfin/src/shortfin/local/systems/host.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/worker.cc b/libshortfin/src/shortfin/local/worker.cc index 42e5c708f..09207e5e4 100644 --- a/libshortfin/src/shortfin/local/worker.cc +++ b/libshortfin/src/shortfin/local/worker.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/local/worker.h b/libshortfin/src/shortfin/local/worker.h index 52f5e5948..73b222b53 100644 --- a/libshortfin/src/shortfin/local/worker.h +++ b/libshortfin/src/shortfin/local/worker.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -79,6 +79,7 @@ class SHORTFIN_API Worker { const Options &options() const { return options_; } const std::string_view name() const { return options_.name; } + iree_loop_t loop() { return loop_; } std::string to_s(); // Gets the Worker that is active for the current thread or nullptr if none. diff --git a/libshortfin/src/shortfin/support/CMakeLists.txt b/libshortfin/src/shortfin/support/CMakeLists.txt index cbe6df89b..895c62052 100644 --- a/libshortfin/src/shortfin/support/CMakeLists.txt +++ b/libshortfin/src/shortfin/support/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. See # https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: diff --git a/libshortfin/src/shortfin/support/api.h b/libshortfin/src/shortfin/support/api.h index ece30d415..78bb513e0 100644 --- a/libshortfin/src/shortfin/support/api.h +++ b/libshortfin/src/shortfin/support/api.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/blocking_executor.cc b/libshortfin/src/shortfin/support/blocking_executor.cc index fde3cc593..7147a0888 100644 --- a/libshortfin/src/shortfin/support/blocking_executor.cc +++ b/libshortfin/src/shortfin/support/blocking_executor.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/blocking_executor.h b/libshortfin/src/shortfin/support/blocking_executor.h index 45395730e..a0361c753 100644 --- a/libshortfin/src/shortfin/support/blocking_executor.h +++ b/libshortfin/src/shortfin/support/blocking_executor.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/blocking_executor_test.cc b/libshortfin/src/shortfin/support/blocking_executor_test.cc index 92a9b31f5..87375859d 100644 --- a/libshortfin/src/shortfin/support/blocking_executor_test.cc +++ b/libshortfin/src/shortfin/support/blocking_executor_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/globals.cc b/libshortfin/src/shortfin/support/globals.cc index 58578e3ec..a063a2bfa 100644 --- a/libshortfin/src/shortfin/support/globals.cc +++ b/libshortfin/src/shortfin/support/globals.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/globals.h b/libshortfin/src/shortfin/support/globals.h index 8af2d47a0..95a814c22 100644 --- a/libshortfin/src/shortfin/support/globals.h +++ b/libshortfin/src/shortfin/support/globals.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/iree_concurrency.h b/libshortfin/src/shortfin/support/iree_concurrency.h index 28ef1e99b..2f355ded6 100644 --- a/libshortfin/src/shortfin/support/iree_concurrency.h +++ b/libshortfin/src/shortfin/support/iree_concurrency.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -16,21 +16,7 @@ namespace shortfin::iree { -namespace detail { -struct thread_ptr_helper { - static void steal(iree_thread_t *obj) { LogIREESteal("iree_thread_t", obj); } - static void retain(iree_thread_t *obj) { - LogIREERetain("iree_thread_t", obj); - iree_thread_retain(obj); - } - static void release(iree_thread_t *obj) { - LogIREERelease("iree_thread_t", obj); - iree_thread_release(obj); - } -}; -}; // namespace detail - -using thread_ptr = object_ptr; +SHORTFIN_IREE_DEF_PTR(thread); // Wraps an iree::slim_mutex as an RAII object. class slim_mutex { diff --git a/libshortfin/src/shortfin/support/iree_concurrency_test.cc b/libshortfin/src/shortfin/support/iree_concurrency_test.cc index 09ca985d7..a7c6bcf0f 100644 --- a/libshortfin/src/shortfin/support/iree_concurrency_test.cc +++ b/libshortfin/src/shortfin/support/iree_concurrency_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/iree_helpers.cc b/libshortfin/src/shortfin/support/iree_helpers.cc index d518e99c3..417a9f443 100644 --- a/libshortfin/src/shortfin/support/iree_helpers.cc +++ b/libshortfin/src/shortfin/support/iree_helpers.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -82,7 +82,9 @@ void SHORTFIN_API LogLiveRefs() { } // namespace detail error::error(std::string message, iree_status_t failing_status) - : message_(std::move(message)), failing_status_(failing_status) { + : code_(iree_status_code(failing_status)), + message_(std::move(message)), + failing_status_(failing_status) { message_.append(": "); } error::error(iree_status_t failing_status) : failing_status_(failing_status) {} diff --git a/libshortfin/src/shortfin/support/iree_helpers.h b/libshortfin/src/shortfin/support/iree_helpers.h index c77ddbaa8..7f2e28cb2 100644 --- a/libshortfin/src/shortfin/support/iree_helpers.h +++ b/libshortfin/src/shortfin/support/iree_helpers.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -15,6 +15,7 @@ #include "iree/hal/api.h" #include "iree/modules/hal/types.h" #include "iree/vm/api.h" +#include "iree/vm/ref_cc.h" #include "shortfin/support/api.h" #if !defined(SHORTFIN_IREE_LOG_RC) @@ -32,6 +33,10 @@ inline std::string_view to_string_view(iree_string_view_t isv) { return std::string_view(isv.data, isv.size); } +inline iree_string_view_t to_iree_string_view(std::string_view sv) { + return iree_make_string_view(sv.data(), sv.size()); +} + namespace iree { // -------------------------------------------------------------------------- // @@ -52,132 +57,6 @@ inline void LogIREESteal(const char *type_name, void *ptr) {} inline void LogLiveRefs() {} #endif -struct hal_buffer_ptr_helper { - static void steal(iree_hal_buffer_t *obj) { - LogIREESteal("iree_hal_buffer_t", obj); - } - static void retain(iree_hal_buffer_t *obj) { - LogIREERetain("iree_hal_buffer_t", obj); - iree_hal_buffer_retain(obj); - } - static void release(iree_hal_buffer_t *obj) { - LogIREERelease("iree_hal_buffer_t", obj); - iree_hal_buffer_release(obj); - } -}; - -struct hal_command_buffer_helper { - static void steal(iree_hal_command_buffer_t *obj) { - LogIREESteal("iree_hal_command_buffer_t", obj); - } - static void retain(iree_hal_command_buffer_t *obj) { - LogIREERetain("iree_hal_command_buffer_t", obj); - iree_hal_command_buffer_retain(obj); - } - static void release(iree_hal_command_buffer_t *obj) { - LogIREERelease("iree_hal_command_buffer_t", obj); - iree_hal_command_buffer_release(obj); - } -}; - -struct hal_device_ptr_helper { - static void steal(iree_hal_device_t *obj) { - LogIREESteal("iree_hal_device_t", obj); - } - static void retain(iree_hal_device_t *obj) { - LogIREERetain("iree_hal_device_t", obj); - iree_hal_device_retain(obj); - } - static void release(iree_hal_device_t *obj) { - LogIREERelease("iree_hal_device_t", obj); - iree_hal_device_release(obj); - } -}; - -struct hal_driver_ptr_helper { - static void steal(iree_hal_driver_t *obj) { - LogIREESteal("iree_hal_driver_t", obj); - } - static void retain(iree_hal_driver_t *obj) { - LogIREERetain("iree_hal_driver_t", obj); - iree_hal_driver_retain(obj); - } - static void release(iree_hal_driver_t *obj) { - LogIREERelease("iree_hal_driver_t", obj); - iree_hal_driver_release(obj); - } -}; - -struct hal_fence_ptr_helper { - static void steal(iree_hal_fence_t *obj) { - LogIREESteal("iree_hal_fence_t", obj); - } - static void retain(iree_hal_fence_t *obj) { - LogIREERetain("iree_hal_fence_t", obj); - iree_hal_fence_retain(obj); - } - static void release(iree_hal_fence_t *obj) { - LogIREERelease("iree_hal_fence_t", obj); - iree_hal_fence_release(obj); - } -}; - -struct hal_semaphore_ptr_helper { - static void steal(iree_hal_semaphore_t *obj) { - LogIREESteal("iree_hal_semaphore_t", obj); - } - static void retain(iree_hal_semaphore_t *obj) { - LogIREERetain("iree_hal_semaphore_t", obj); - iree_hal_semaphore_retain(obj); - } - static void release(iree_hal_semaphore_t *obj) { - LogIREERelease("iree_hal_semaphore_t", obj); - iree_hal_semaphore_release(obj); - } -}; - -struct vm_context_ptr_helper { - static void steal(iree_vm_context_t *obj) { - LogIREESteal("iree_vm_context_t", obj); - } - static void retain(iree_vm_context_t *obj) { - LogIREERetain("iree_vm_context_t", obj); - iree_vm_context_retain(obj); - } - static void release(iree_vm_context_t *obj) { - LogIREERelease("iree_vm_context_t", obj); - iree_vm_context_release(obj); - } -}; - -struct vm_instance_ptr_helper { - static void steal(iree_vm_instance_t *obj) { - LogIREESteal("iree_vm_instance_t", obj); - } - static void retain(iree_vm_instance_t *obj) { - LogIREERetain("iree_vm_instance_t", obj); - iree_vm_instance_retain(obj); - } - static void release(iree_vm_instance_t *obj) { - LogIREERelease("iree_vm_instance_t", obj); - iree_vm_instance_release(obj); - } -}; - -struct vm_module_ptr_helper { - static void steal(iree_vm_module_t *obj) { - LogIREESteal("iree_vm_module_t", obj); - } - static void retain(iree_vm_module_t *obj) { - LogIREERetain("iree_vm_module_t", obj); - iree_vm_module_retain(obj); - } - static void release(iree_vm_module_t *obj) { - LogIREERelease("iree_vm_module_t", obj); - iree_vm_module_release(obj); - } -}; - }; // namespace detail // Wraps an IREE retain/release style object pointer in a smart-pointer @@ -261,24 +140,39 @@ class object_ptr { friend class Assignment; }; -using hal_buffer_ptr = - object_ptr; -using hal_command_buffer_ptr = - object_ptr; -using hal_driver_ptr = - object_ptr; -using hal_device_ptr = - object_ptr; -using hal_fence_ptr = - object_ptr; -using hal_semaphore_ptr = - object_ptr; -using vm_context_ptr = - object_ptr; -using vm_instance_ptr = - object_ptr; -using vm_module_ptr = - object_ptr; +// Defines a reference counting helper struct named like +// iree_hal_buffer_ptr_helper (for type_stem == hal_buffer). +// These must be defined in the shortfin::iree::detail namespace. +#define SHORTFIN_IREE_DEF_PTR(type_stem) \ + namespace detail { \ + struct type_stem##_ptr_helper { \ + static void steal(iree_##type_stem##_t *obj) { \ + LogIREESteal(#type_stem "_t", obj); \ + } \ + static void retain(iree_##type_stem##_t *obj) { \ + LogIREERetain(#type_stem "_t", obj); \ + iree_##type_stem##_retain(obj); \ + } \ + static void release(iree_##type_stem##_t *obj) { \ + LogIREERelease(#type_stem "_t", obj); \ + iree_##type_stem##_release(obj); \ + } \ + }; \ + } \ + using type_stem##_ptr = \ + object_ptr + +SHORTFIN_IREE_DEF_PTR(hal_command_buffer); +SHORTFIN_IREE_DEF_PTR(hal_buffer); +SHORTFIN_IREE_DEF_PTR(hal_buffer_view); +SHORTFIN_IREE_DEF_PTR(hal_device); +SHORTFIN_IREE_DEF_PTR(hal_driver); +SHORTFIN_IREE_DEF_PTR(hal_fence); +SHORTFIN_IREE_DEF_PTR(hal_semaphore); +SHORTFIN_IREE_DEF_PTR(vm_context); +SHORTFIN_IREE_DEF_PTR(vm_instance); +SHORTFIN_IREE_DEF_PTR(vm_list); +SHORTFIN_IREE_DEF_PTR(vm_module); // Holds a pointer allocated by some allocator, deleting it if still owned // at destruction time. @@ -386,8 +280,11 @@ class SHORTFIN_API error : public std::exception { return message_.c_str(); }; + iree_status_code_t code() const { return code_; } + private: void AppendStatus() const noexcept; + iree_status_code_t code_; mutable std::string message_; mutable iree_status_t failing_status_; mutable bool status_appended_ = false; @@ -432,13 +329,27 @@ class ignorable_status { ignorable_status(ignorable_status &&other) = delete; ~ignorable_status() { iree_status_ignore(status_); } - operator iree_status_t() const { return status_; } + // Consumes that status. Only the first consumer will receive all payloads. + // Others will just get the cloned basic status. + iree_status_t ConsumeStatus() { + iree_status_t local_status = status_; + status_ = iree_status_clone(status_); + return local_status; + } iree_status_t status() const { return status_; } private: - iree_status_t status_; + mutable iree_status_t status_; }; +// -------------------------------------------------------------------------- // +// VM Ref and Variant Interop +// -------------------------------------------------------------------------- // + +using vm_opaque_ref = ::iree::vm::opaque_ref; +template +using vm_ref = ::iree::vm::ref; + } // namespace iree } // namespace shortfin diff --git a/libshortfin/src/shortfin/support/iree_helpers_test.cc b/libshortfin/src/shortfin/support/iree_helpers_test.cc index bf059ee98..5f4adcdaa 100644 --- a/libshortfin/src/shortfin/support/iree_helpers_test.cc +++ b/libshortfin/src/shortfin/support/iree_helpers_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/logging.cc b/libshortfin/src/shortfin/support/logging.cc index 5d9569de7..1b2ff56b5 100644 --- a/libshortfin/src/shortfin/support/logging.cc +++ b/libshortfin/src/shortfin/support/logging.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/logging.h b/libshortfin/src/shortfin/support/logging.h index 337ebacae..99d9c64e8 100644 --- a/libshortfin/src/shortfin/support/logging.h +++ b/libshortfin/src/shortfin/support/logging.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -13,6 +13,14 @@ #define SHORTFIN_LOG_LIFETIMES 0 #endif +// Scheduler logging. +#define SHORTFIN_SCHED_LOG_ENABLED 0 +#if SHORTFIN_SCHED_LOG_ENABLED +#define SHORTFIN_SCHED_LOG(...) shortfin::logging::info("SCHED: " __VA_ARGS__) +#else +#define SHORTFIN_SCHED_LOG(...) +#endif + namespace shortfin::logging { // TODO: Re-export doesn't really work like this. Need to define API diff --git a/libshortfin/src/shortfin/support/stl_extras.h b/libshortfin/src/shortfin/support/stl_extras.h index 1c6a1380d..5f87ae566 100644 --- a/libshortfin/src/shortfin/support/stl_extras.h +++ b/libshortfin/src/shortfin/support/stl_extras.h @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/src/shortfin/support/stl_extras_test.cc b/libshortfin/src/shortfin/support/stl_extras_test.cc index cc558c3d4..5335b5b1b 100644 --- a/libshortfin/src/shortfin/support/stl_extras_test.cc +++ b/libshortfin/src/shortfin/support/stl_extras_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/tests/amdgpu_system_test.py b/libshortfin/tests/amdgpu_system_test.py index 4b887ea54..cf04a7caf 100644 --- a/libshortfin/tests/amdgpu_system_test.py +++ b/libshortfin/tests/amdgpu_system_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/tests/api/array_storage_test.py b/libshortfin/tests/api/array_storage_test.py new file mode 100644 index 000000000..9af208f5a --- /dev/null +++ b/libshortfin/tests/api/array_storage_test.py @@ -0,0 +1,185 @@ +# Copyright 2024 Advanced Micro Devices, Inc. +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import pytest + +import shortfin as sf +import shortfin.array as sfnp + + +@pytest.fixture +def lsys(): + sc = sf.host.CPUSystemBuilder() + lsys = sc.create_system() + yield lsys + lsys.shutdown() + + +@pytest.fixture +def scope(lsys): + return lsys.create_scope() + + +@pytest.fixture +def device(scope): + return scope.device(0) + + +def test_allocate_host(device): + s = sfnp.storage.allocate_host(device, 32) + assert len(bytes(s.data)) == 32 + + +def test_allocate_device(device): + s = sfnp.storage.allocate_device(device, 64) + assert len(bytes(s.data)) == 64 + + +def test_fill1(lsys, device): + async def main(): + s = sfnp.storage.allocate_host(device, 8) + s.fill(b"0") + await device + assert bytes(s.data) == b"00000000" + + lsys.run(main()) + + +def test_fill2(lsys, device): + async def main(): + s = sfnp.storage.allocate_host(device, 8) + s.fill(b"01") + await device + assert bytes(s.data) == b"01010101" + + lsys.run(main()) + + +def test_fill4(lsys, device): + async def main(): + s = sfnp.storage.allocate_host(device, 8) + s.fill(b"0123") + await device + assert bytes(s.data) == b"01230123" + + lsys.run(main()) + + +def test_fill_error(device): + s = sfnp.storage.allocate_host(device, 8) + with pytest.raises(RuntimeError): + s.fill(b"") + with pytest.raises(RuntimeError): + s.fill(b"012") + with pytest.raises(RuntimeError): + s.fill(b"01234") + with pytest.raises(RuntimeError): + s.fill(b"01234567") + + +@pytest.mark.parametrize( + "pattern,size", + [ + (b"", 8), + (b"012", 8), + (b"01234", 8), + (b"01234567", 8), + ], +) +def test_fill_error(lsys, device, pattern, size): + async def main(): + src = sfnp.storage.allocate_host(device, size) + src.fill(pattern) + + with pytest.raises( + ValueError, match="fill value length is not one of the supported values" + ): + lsys.run(main()) + + +def test_map_read(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + src.fill(b"0123") + await device + with src.map(read=True) as m: + assert m.valid + assert bytes(m) == b"01230123" + + lsys.run(main()) + + +def test_map_read_not_writable(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + src.fill(b"0123") + await device + with src.map(read=True) as m: + mv = memoryview(m) + assert mv.readonly + mv[0] = ord(b"9") + + with pytest.raises(TypeError, match="cannot modify"): + lsys.run(main()) + + +def test_map_write(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + src.fill(b"0123") + await device + with src.map(read=True, write=True) as m: + mv = memoryview(m) + assert not mv.readonly + mv[0] = ord(b"9") + assert bytes(src.data) == b"91230123" + + lsys.run(main()) + + +def test_map_discard(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + src.fill(b"0123") + await device + with src.map(write=True, discard=True) as m: + mv = memoryview(m) + assert not mv.readonly + for i in range(8): + mv[i] = ord(b"9") - i + assert bytes(src.data) == b"98765432" + + lsys.run(main()) + + +def test_data_write(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + src.data = b"98765432" + assert bytes(src.data) == b"98765432" + + lsys.run(main()) + + +def test_mapping_explicit_close(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + m = src.map(write=True, discard=True) + assert m.valid + m.close() + assert not m.valid + + lsys.run(main()) + + +def test_mapping_context_manager(lsys, device): + async def main(): + src = sfnp.storage.allocate_host(device, 8) + with src.map(write=True, discard=True) as m: + assert m.valid + assert not m.valid + + lsys.run(main()) diff --git a/libshortfin/tests/api/array_test.py b/libshortfin/tests/api/array_test.py new file mode 100644 index 000000000..729f091d8 --- /dev/null +++ b/libshortfin/tests/api/array_test.py @@ -0,0 +1,123 @@ +# Copyright 2024 Advanced Micro Devices, Inc. +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import array +import pytest + +import shortfin as sf +import shortfin.array as sfnp + + +@pytest.fixture +def lsys(): + sc = sf.host.CPUSystemBuilder() + lsys = sc.create_system() + yield lsys + lsys.shutdown() + + +@pytest.fixture +def scope(lsys): + return lsys.create_scope() + + +@pytest.fixture +def device(scope): + return scope.device(0) + + +def test_storage_constructor(lsys, device): + async def main(): + s = sfnp.storage.allocate_host(device, 8) + s.fill(b"\0\1\2\3") + await device + ary = sfnp.device_array(s, [2, 4], sfnp.uint8) + assert ary.dtype == sfnp.uint8 + assert ary.shape == [2, 4] + assert str(ary) == "{{0, 1, 2, 3},\n {0, 1, 2, 3}}" + assert ary.device == device + assert ary.storage == s + + lsys.run(main()) + + +def test_device_constructor(lsys, device): + async def main(): + ary = sfnp.device_array(device, [2, 4], sfnp.uint8) + ary.storage.fill(b"\0\1\2\3") + await device + assert ary.dtype == sfnp.uint8 + assert ary.shape == [2, 4] + assert str(ary) == "{{0, 1, 2, 3},\n {0, 1, 2, 3}}" + assert ary.device == device + + lsys.run(main()) + + +def test_fill_copy_from_for_transfer(lsys, device): + async def main(): + src = sfnp.device_array(device, [2, 4], sfnp.uint8) + src.fill(b"\0\1\2\3") + dst = src.for_transfer() + dst.copy_from(src) + await device + assert str(dst) == "{{0, 1, 2, 3},\n {0, 1, 2, 3}}" + + lsys.run(main()) + + +def test_fill_copy_to_for_transfer(lsys, device): + async def main(): + src = sfnp.device_array(device, [2, 4], sfnp.uint8) + src.fill(b"\0\1\2\3") + dst = src.for_transfer() + src.copy_to(dst) + await device + assert str(dst) == "{{0, 1, 2, 3},\n {0, 1, 2, 3}}" + + lsys.run(main()) + + +def test_shape_overflow(lsys, device): + async def main(): + s = sfnp.storage.allocate_host(device, 4) + _ = sfnp.device_array(s, [2, 4], sfnp.uint8) + + with pytest.raises( + ValueError, match="Array storage requires at least 8 bytes but has only 4" + ): + lsys.run(main()) + + +@pytest.mark.parametrize( + "dtype,code,py_value,expected_str", + [ + (sfnp.int8, "b", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), + (sfnp.int16, "h", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), + (sfnp.int32, "i", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), + ( + sfnp.float32, + "f", + 42.0, + "{{ 42., 42., 42., 42.},\n { 42., 42., 42., 42.}}", + ), + ( + sfnp.float64, + "d", + 42.0, + "{{ 42., 42., 42., 42.},\n { 42., 42., 42., 42.}}", + ), + ], +) +def test_xtensor_types(scope, dtype, code, py_value, expected_str): + ary = sfnp.device_array.for_host(scope.device(0), [2, 4], dtype) + ary.storage.data = array.array(code, [py_value] * 8) + s = str(ary) + print("__str__ =", s) + assert expected_str == s, f"Expected '{expected_str}' == '{s}'" + r = repr(ary) + print("__repr__ =", r) + assert expected_str in r, f"Expected '{expected_str}' in '{r}'" diff --git a/libshortfin/tests/array_test.py b/libshortfin/tests/array_test.py deleted file mode 100644 index 9f53da1c3..000000000 --- a/libshortfin/tests/array_test.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2024 Advanced Micro Devices, Inc -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import array -import pytest -import time - -import shortfin as sf - - -@pytest.fixture -def lsys(): - sc = sf.host.CPUSystemBuilder() - lsys = sc.create_system() - yield lsys - lsys.shutdown() - - -@pytest.fixture -def scope(lsys): - # TODO: Should adopt the main thread. - worker = lsys.create_worker("main") - return lsys.create_scope(worker) - - -def test_storage(scope): - storage = sf.array.storage.allocate_host(scope.device(0), 32) - print(storage) - ary = sf.array.device_array(storage, [2, 4], sf.array.float32) - print(ary) - print(ary.shape) - assert ary.shape == [2, 4] - assert ary.dtype == sf.array.float32 - - print("ARY.DEVICE=", ary.device, ary.device.__class__) - print("SCOPE.DEVICE=", scope.device(0)) - print("EQ:", ary.device == scope.device(0)) - - assert ary.device == scope.device(0) - - # Mapping API contract. - with storage.map(read=True) as m: - assert m.valid - mv = memoryview(m) - assert len(mv) == 32 - assert not m.valid - - storage.data = array.array("f", [1.234534523] * 8) - print("WRITTEN:", ary) - - read_back = array.array("f") - read_back.frombytes(storage.data) - print("READ BACK:", read_back) - - -@pytest.mark.parametrize( - "dtype,code,py_value,expected_repr", - [ - (sf.array.int8, "b", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), - (sf.array.int16, "h", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), - (sf.array.int32, "i", 42, "{{42, 42, 42, 42},\n {42, 42, 42, 42}}"), - ( - sf.array.float32, - "f", - 42.0, - "{{ 42., 42., 42., 42.},\n { 42., 42., 42., 42.}}", - ), - ( - sf.array.float64, - "d", - 42.0, - "{{ 42., 42., 42., 42.},\n { 42., 42., 42., 42.}}", - ), - ], -) -def test_xtensor_types(scope, dtype, code, py_value, expected_repr): - ary = sf.array.device_array.for_host(scope.device(0), [2, 4], dtype) - ary.storage.data = array.array(code, [py_value] * 8) - r = repr(ary) - print(r) - assert expected_repr in r, f"Expected '{expected_repr}' in '{r}'" - - -def test_device_array(scope): - ary1 = sf.array.device_array(scope.device(0), [32, 1, 4], sf.array.float32) - print(ary1) - assert ary1.shape == [32, 1, 4] - assert ary1.dtype == sf.array.float32 - assert scope.device(0) == ary1.device - - hary1 = sf.array.device_array.for_transfer(ary1) - print(hary1) - assert isinstance(hary1, sf.array.device_array) - assert hary1.shape == ary1.shape - assert hary1.dtype == ary1.dtype - assert hary1.device == ary1.device - - -def test_device_array_fill(scope): - ary1 = sf.array.device_array(scope.device(0), [32, 1, 4], sf.array.int32) - ary1.storage.fill(array.array("i", [42])) - # TODO: Transfer to host and verify. diff --git a/libshortfin/tests/examples/async_test.py b/libshortfin/tests/examples/async_test.py index 1595d7d8e..ea3dc20ab 100644 --- a/libshortfin/tests/examples/async_test.py +++ b/libshortfin/tests/examples/async_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/tests/examples/fastapi_test.py b/libshortfin/tests/examples/fastapi_test.py index f19c1c12f..5640f0d4b 100644 --- a/libshortfin/tests/examples/fastapi_test.py +++ b/libshortfin/tests/examples/fastapi_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/tests/host_cpu_system_test.py b/libshortfin/tests/host_cpu_system_test.py index 03a10da0c..1a37f02c1 100644 --- a/libshortfin/tests/host_cpu_system_test.py +++ b/libshortfin/tests/host_cpu_system_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/libshortfin/tests/local_scope_test.py b/libshortfin/tests/local_scope_test.py index 9f56e7833..379f13d4b 100644 --- a/libshortfin/tests/local_scope_test.py +++ b/libshortfin/tests/local_scope_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/conftest.py b/sharktank/conftest.py index b149a190b..fd9c47447 100644 --- a/sharktank/conftest.py +++ b/sharktank/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/integration/models/punet/integration_test.py b/sharktank/integration/models/punet/integration_test.py index 773102537..182b37a50 100644 --- a/sharktank/integration/models/punet/integration_test.py +++ b/sharktank/integration/models/punet/integration_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/setup.py b/sharktank/setup.py index f63c4d98e..b8caf9e7d 100644 --- a/sharktank/setup.py +++ b/sharktank/setup.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/__init__.py b/sharktank/sharktank/__init__.py index e243f3c35..a85ba359d 100644 --- a/sharktank/sharktank/__init__.py +++ b/sharktank/sharktank/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/export_paged_llm_v1.py b/sharktank/sharktank/examples/export_paged_llm_v1.py index 0a75ea742..44c9d88ce 100644 --- a/sharktank/sharktank/examples/export_paged_llm_v1.py +++ b/sharktank/sharktank/examples/export_paged_llm_v1.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/paged_llm_v1.py b/sharktank/sharktank/examples/paged_llm_v1.py index a9d14195c..16b309e23 100644 --- a/sharktank/sharktank/examples/paged_llm_v1.py +++ b/sharktank/sharktank/examples/paged_llm_v1.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/sharding/export_ffn_net.py b/sharktank/sharktank/examples/sharding/export_ffn_net.py index 2dcbffc75..f80b9a2ac 100644 --- a/sharktank/sharktank/examples/sharding/export_ffn_net.py +++ b/sharktank/sharktank/examples/sharding/export_ffn_net.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/sharding/shard_llm_dataset.py b/sharktank/sharktank/examples/sharding/shard_llm_dataset.py index 9b1585f18..0921a2c83 100644 --- a/sharktank/sharktank/examples/sharding/shard_llm_dataset.py +++ b/sharktank/sharktank/examples/sharding/shard_llm_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/validate_direct_llama_model.py b/sharktank/sharktank/examples/validate_direct_llama_model.py index 52ef7c9eb..d4cb7288a 100644 --- a/sharktank/sharktank/examples/validate_direct_llama_model.py +++ b/sharktank/sharktank/examples/validate_direct_llama_model.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/validate_llama_ref_model.py b/sharktank/sharktank/examples/validate_llama_ref_model.py index 6863e861e..0ce59b3e9 100644 --- a/sharktank/sharktank/examples/validate_llama_ref_model.py +++ b/sharktank/sharktank/examples/validate_llama_ref_model.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/examples/validate_paged_llama_model.py b/sharktank/sharktank/examples/validate_paged_llama_model.py index c005a3fdb..a26ddbf31 100644 --- a/sharktank/sharktank/examples/validate_paged_llama_model.py +++ b/sharktank/sharktank/examples/validate_paged_llama_model.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/__init__.py b/sharktank/sharktank/kernels/__init__.py index 61466d31b..308e20ef4 100644 --- a/sharktank/sharktank/kernels/__init__.py +++ b/sharktank/sharktank/kernels/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/attention.py b/sharktank/sharktank/kernels/attention.py index a0a5662a1..3e2ef4a57 100644 --- a/sharktank/sharktank/kernels/attention.py +++ b/sharktank/sharktank/kernels/attention.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/base.py b/sharktank/sharktank/kernels/base.py index 02b4c28ec..8c99c81d9 100644 --- a/sharktank/sharktank/kernels/base.py +++ b/sharktank/sharktank/kernels/base.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/batch_matmul_transpose_b.py b/sharktank/sharktank/kernels/batch_matmul_transpose_b.py index c99bf00d2..11a6b5fc2 100644 --- a/sharktank/sharktank/kernels/batch_matmul_transpose_b.py +++ b/sharktank/sharktank/kernels/batch_matmul_transpose_b.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/conv_2d_nchw_fchw.py b/sharktank/sharktank/kernels/conv_2d_nchw_fchw.py index 76ab18670..9ada3b099 100644 --- a/sharktank/sharktank/kernels/conv_2d_nchw_fchw.py +++ b/sharktank/sharktank/kernels/conv_2d_nchw_fchw.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/mmt_block_scaled_offset_q4.py b/sharktank/sharktank/kernels/mmt_block_scaled_offset_q4.py index 972194c83..2ed171115 100644 --- a/sharktank/sharktank/kernels/mmt_block_scaled_offset_q4.py +++ b/sharktank/sharktank/kernels/mmt_block_scaled_offset_q4.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/mmt_block_scaled_q8.py b/sharktank/sharktank/kernels/mmt_block_scaled_q8.py index 1ea2cb467..d589e38e4 100644 --- a/sharktank/sharktank/kernels/mmt_block_scaled_q8.py +++ b/sharktank/sharktank/kernels/mmt_block_scaled_q8.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/mmt_super_block_scaled_offset_q4.py b/sharktank/sharktank/kernels/mmt_super_block_scaled_offset_q4.py index b4ce5b7a6..e557b9476 100644 --- a/sharktank/sharktank/kernels/mmt_super_block_scaled_offset_q4.py +++ b/sharktank/sharktank/kernels/mmt_super_block_scaled_offset_q4.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/mmtfp.py b/sharktank/sharktank/kernels/mmtfp.py index 3ca003c0c..de7e4d0af 100644 --- a/sharktank/sharktank/kernels/mmtfp.py +++ b/sharktank/sharktank/kernels/mmtfp.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/pooling_nchw_sum.py b/sharktank/sharktank/kernels/pooling_nchw_sum.py index 86a9fbf5e..f9bedf121 100644 --- a/sharktank/sharktank/kernels/pooling_nchw_sum.py +++ b/sharktank/sharktank/kernels/pooling_nchw_sum.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/batch_matmul_transpose_b.mlir b/sharktank/sharktank/kernels/templates/batch_matmul_transpose_b.mlir index 21fd15773..908ca1c7f 100644 --- a/sharktank/sharktank/kernels/templates/batch_matmul_transpose_b.mlir +++ b/sharktank/sharktank/kernels/templates/batch_matmul_transpose_b.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/conv_2d_nchw_fchw.mlir b/sharktank/sharktank/kernels/templates/conv_2d_nchw_fchw.mlir index e0bc8bee2..8f53d0643 100644 --- a/sharktank/sharktank/kernels/templates/conv_2d_nchw_fchw.mlir +++ b/sharktank/sharktank/kernels/templates/conv_2d_nchw_fchw.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/flash_attention.mlir b/sharktank/sharktank/kernels/templates/flash_attention.mlir index a95c204fc..db76db84f 100644 --- a/sharktank/sharktank/kernels/templates/flash_attention.mlir +++ b/sharktank/sharktank/kernels/templates/flash_attention.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/mmt_block_scaled_offset_q4_unsigned.mlir b/sharktank/sharktank/kernels/templates/mmt_block_scaled_offset_q4_unsigned.mlir index a1fe684c6..a7f3138cb 100644 --- a/sharktank/sharktank/kernels/templates/mmt_block_scaled_offset_q4_unsigned.mlir +++ b/sharktank/sharktank/kernels/templates/mmt_block_scaled_offset_q4_unsigned.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/mmt_block_scaled_q8_3d.mlir b/sharktank/sharktank/kernels/templates/mmt_block_scaled_q8_3d.mlir index d71d42a31..06064fc05 100644 --- a/sharktank/sharktank/kernels/templates/mmt_block_scaled_q8_3d.mlir +++ b/sharktank/sharktank/kernels/templates/mmt_block_scaled_q8_3d.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/mmt_super_block_scaled_offset_q4_unsigned_3d.mlir b/sharktank/sharktank/kernels/templates/mmt_super_block_scaled_offset_q4_unsigned_3d.mlir index 35b55840b..3447ee877 100644 --- a/sharktank/sharktank/kernels/templates/mmt_super_block_scaled_offset_q4_unsigned_3d.mlir +++ b/sharktank/sharktank/kernels/templates/mmt_super_block_scaled_offset_q4_unsigned_3d.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/mmtfp_2d.mlir b/sharktank/sharktank/kernels/templates/mmtfp_2d.mlir index aeb512c85..177ed269d 100644 --- a/sharktank/sharktank/kernels/templates/mmtfp_2d.mlir +++ b/sharktank/sharktank/kernels/templates/mmtfp_2d.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/mmtfp_3d.mlir b/sharktank/sharktank/kernels/templates/mmtfp_3d.mlir index 0167e24fb..ccc71ead4 100644 --- a/sharktank/sharktank/kernels/templates/mmtfp_3d.mlir +++ b/sharktank/sharktank/kernels/templates/mmtfp_3d.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/kernels/templates/pooling_nchw_sum.mlir b/sharktank/sharktank/kernels/templates/pooling_nchw_sum.mlir index d43ceb6bc..df2036b5f 100644 --- a/sharktank/sharktank/kernels/templates/pooling_nchw_sum.mlir +++ b/sharktank/sharktank/kernels/templates/pooling_nchw_sum.mlir @@ -1,4 +1,4 @@ -// Copyright 2024 Advanced Micro Devices, Inc +// Copyright 2024 Advanced Micro Devices, Inc. // // Licensed under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/__init__.py b/sharktank/sharktank/layers/__init__.py index bb51a99ed..25a00c80f 100644 --- a/sharktank/sharktank/layers/__init__.py +++ b/sharktank/sharktank/layers/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/base.py b/sharktank/sharktank/layers/base.py index 78936ca2c..90c976f25 100644 --- a/sharktank/sharktank/layers/base.py +++ b/sharktank/sharktank/layers/base.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/causal_llm.py b/sharktank/sharktank/layers/causal_llm.py index d253af617..3d91683fe 100644 --- a/sharktank/sharktank/layers/causal_llm.py +++ b/sharktank/sharktank/layers/causal_llm.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/configs/__init__.py b/sharktank/sharktank/layers/configs/__init__.py index 2e1ef5d42..21336d1d2 100644 --- a/sharktank/sharktank/layers/configs/__init__.py +++ b/sharktank/sharktank/layers/configs/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/configs/llm_configs.py b/sharktank/sharktank/layers/configs/llm_configs.py index ab548e286..a9aad8088 100644 --- a/sharktank/sharktank/layers/configs/llm_configs.py +++ b/sharktank/sharktank/layers/configs/llm_configs.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/conv.py b/sharktank/sharktank/layers/conv.py index 4c90de2a5..e2cb97002 100644 --- a/sharktank/sharktank/layers/conv.py +++ b/sharktank/sharktank/layers/conv.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/kv_cache.py b/sharktank/sharktank/layers/kv_cache.py index 319f2353c..47b465bd2 100644 --- a/sharktank/sharktank/layers/kv_cache.py +++ b/sharktank/sharktank/layers/kv_cache.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/linear.py b/sharktank/sharktank/layers/linear.py index 926664d18..a27123d4b 100644 --- a/sharktank/sharktank/layers/linear.py +++ b/sharktank/sharktank/layers/linear.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/norm.py b/sharktank/sharktank/layers/norm.py index 68818a2d7..b9ae4b41a 100644 --- a/sharktank/sharktank/layers/norm.py +++ b/sharktank/sharktank/layers/norm.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/rotary_embedding.py b/sharktank/sharktank/layers/rotary_embedding.py index 18984713d..2e47ca5c7 100644 --- a/sharktank/sharktank/layers/rotary_embedding.py +++ b/sharktank/sharktank/layers/rotary_embedding.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/layers/token_embedding.py b/sharktank/sharktank/layers/token_embedding.py index aeeb34127..32e7fec8f 100644 --- a/sharktank/sharktank/layers/token_embedding.py +++ b/sharktank/sharktank/layers/token_embedding.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/llama/llama.py b/sharktank/sharktank/models/llama/llama.py index 5bd6d4d48..8be1b46ea 100644 --- a/sharktank/sharktank/models/llama/llama.py +++ b/sharktank/sharktank/models/llama/llama.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/llama/llama_ref.py b/sharktank/sharktank/models/llama/llama_ref.py index 0639f2723..c635565ca 100644 --- a/sharktank/sharktank/models/llama/llama_ref.py +++ b/sharktank/sharktank/models/llama/llama_ref.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/llama/testing.py b/sharktank/sharktank/models/llama/testing.py index 73028a37c..e1f428e93 100644 --- a/sharktank/sharktank/models/llama/testing.py +++ b/sharktank/sharktank/models/llama/testing.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/llama/tools/generate_data.py b/sharktank/sharktank/models/llama/tools/generate_data.py index 9367fc8a6..67b2bac75 100644 --- a/sharktank/sharktank/models/llama/tools/generate_data.py +++ b/sharktank/sharktank/models/llama/tools/generate_data.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/config.py b/sharktank/sharktank/models/punet/config.py index 88f4dca43..4b328f9c2 100644 --- a/sharktank/sharktank/models/punet/config.py +++ b/sharktank/sharktank/models/punet/config.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/layers.py b/sharktank/sharktank/models/punet/layers.py index 4a723b4de..9294deee2 100644 --- a/sharktank/sharktank/models/punet/layers.py +++ b/sharktank/sharktank/models/punet/layers.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/model.py b/sharktank/sharktank/models/punet/model.py index fbc8f6b1e..05c132f44 100644 --- a/sharktank/sharktank/models/punet/model.py +++ b/sharktank/sharktank/models/punet/model.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/sharding.py b/sharktank/sharktank/models/punet/sharding.py index c72fe4914..22237b8bb 100644 --- a/sharktank/sharktank/models/punet/sharding.py +++ b/sharktank/sharktank/models/punet/sharding.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/testing.py b/sharktank/sharktank/models/punet/testing.py index c159b2226..40fe5a0fc 100644 --- a/sharktank/sharktank/models/punet/testing.py +++ b/sharktank/sharktank/models/punet/testing.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/tools/import_brevitas_dataset.py b/sharktank/sharktank/models/punet/tools/import_brevitas_dataset.py index f1ba40bfd..fa862105c 100644 --- a/sharktank/sharktank/models/punet/tools/import_brevitas_dataset.py +++ b/sharktank/sharktank/models/punet/tools/import_brevitas_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/tools/import_hf_dataset.py b/sharktank/sharktank/models/punet/tools/import_hf_dataset.py index fc9f3ddca..0afa5222d 100644 --- a/sharktank/sharktank/models/punet/tools/import_hf_dataset.py +++ b/sharktank/sharktank/models/punet/tools/import_hf_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/tools/run_diffuser_ref.py b/sharktank/sharktank/models/punet/tools/run_diffuser_ref.py index 54aee41cc..8ff965f01 100644 --- a/sharktank/sharktank/models/punet/tools/run_diffuser_ref.py +++ b/sharktank/sharktank/models/punet/tools/run_diffuser_ref.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/tools/run_punet.py b/sharktank/sharktank/models/punet/tools/run_punet.py index 3b99d515e..ace279a3b 100644 --- a/sharktank/sharktank/models/punet/tools/run_punet.py +++ b/sharktank/sharktank/models/punet/tools/run_punet.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/models/punet/tools/sample_data.py b/sharktank/sharktank/models/punet/tools/sample_data.py index 81e560fe3..2346e9a80 100644 --- a/sharktank/sharktank/models/punet/tools/sample_data.py +++ b/sharktank/sharktank/models/punet/tools/sample_data.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/__init__.py b/sharktank/sharktank/ops/__init__.py index dd9464b83..164f087fb 100644 --- a/sharktank/sharktank/ops/__init__.py +++ b/sharktank/sharktank/ops/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/_registry.py b/sharktank/sharktank/ops/_registry.py index c519af75b..3c8ea0aed 100644 --- a/sharktank/sharktank/ops/_registry.py +++ b/sharktank/sharktank/ops/_registry.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/attention_impls.py b/sharktank/sharktank/ops/attention_impls.py index eafb88780..8ffb1f51e 100644 --- a/sharktank/sharktank/ops/attention_impls.py +++ b/sharktank/sharktank/ops/attention_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/custom_impls.py b/sharktank/sharktank/ops/custom_impls.py index d0fa78025..fe6ae27b1 100644 --- a/sharktank/sharktank/ops/custom_impls.py +++ b/sharktank/sharktank/ops/custom_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/default_impls.py b/sharktank/sharktank/ops/default_impls.py index bbc65a9c7..8b009faf9 100644 --- a/sharktank/sharktank/ops/default_impls.py +++ b/sharktank/sharktank/ops/default_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/qconv_impls.py b/sharktank/sharktank/ops/qconv_impls.py index 877c7461d..af1199976 100644 --- a/sharktank/sharktank/ops/qconv_impls.py +++ b/sharktank/sharktank/ops/qconv_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/qlinear_impls.py b/sharktank/sharktank/ops/qlinear_impls.py index 732ac3111..0a381d613 100644 --- a/sharktank/sharktank/ops/qlinear_impls.py +++ b/sharktank/sharktank/ops/qlinear_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/shape.py b/sharktank/sharktank/ops/shape.py index a39d46156..69683f84f 100644 --- a/sharktank/sharktank/ops/shape.py +++ b/sharktank/sharktank/ops/shape.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/sharded_impls.py b/sharktank/sharktank/ops/sharded_impls.py index b1ef57090..ed639915e 100644 --- a/sharktank/sharktank/ops/sharded_impls.py +++ b/sharktank/sharktank/ops/sharded_impls.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/ops/signatures.py b/sharktank/sharktank/ops/signatures.py index 07ae56bb1..6350d3e0b 100644 --- a/sharktank/sharktank/ops/signatures.py +++ b/sharktank/sharktank/ops/signatures.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/tools/compare_safetensors.py b/sharktank/sharktank/tools/compare_safetensors.py index 9ae832d91..888ff0bab 100644 --- a/sharktank/sharktank/tools/compare_safetensors.py +++ b/sharktank/sharktank/tools/compare_safetensors.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/tools/dump_gguf.py b/sharktank/sharktank/tools/dump_gguf.py index 1ddfa19d0..eb8321cb7 100644 --- a/sharktank/sharktank/tools/dump_gguf.py +++ b/sharktank/sharktank/tools/dump_gguf.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/transforms/dataset/__init__.py b/sharktank/sharktank/transforms/dataset/__init__.py index e8c2c6224..b6a2a400a 100644 --- a/sharktank/sharktank/transforms/dataset/__init__.py +++ b/sharktank/sharktank/transforms/dataset/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/transforms/dataset/sharding.py b/sharktank/sharktank/transforms/dataset/sharding.py index 7ec480900..1d21e7b71 100644 --- a/sharktank/sharktank/transforms/dataset/sharding.py +++ b/sharktank/sharktank/transforms/dataset/sharding.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/__init__.py b/sharktank/sharktank/types/__init__.py index f58e3c334..3e3bbb3d5 100644 --- a/sharktank/sharktank/types/__init__.py +++ b/sharktank/sharktank/types/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/gguf_interop/__init__.py b/sharktank/sharktank/types/gguf_interop/__init__.py index b5bfa4a98..3e240e8fd 100644 --- a/sharktank/sharktank/types/gguf_interop/__init__.py +++ b/sharktank/sharktank/types/gguf_interop/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/gguf_interop/base.py b/sharktank/sharktank/types/gguf_interop/base.py index a343e333c..67bfe8743 100644 --- a/sharktank/sharktank/types/gguf_interop/base.py +++ b/sharktank/sharktank/types/gguf_interop/base.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/gguf_interop/layouts.py b/sharktank/sharktank/types/gguf_interop/layouts.py index 2c7f50f4a..9dcdd4283 100644 --- a/sharktank/sharktank/types/gguf_interop/layouts.py +++ b/sharktank/sharktank/types/gguf_interop/layouts.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/layout_utils.py b/sharktank/sharktank/types/layout_utils.py index 4e5982e7d..1e5f8db08 100644 --- a/sharktank/sharktank/types/layout_utils.py +++ b/sharktank/sharktank/types/layout_utils.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/layouts.py b/sharktank/sharktank/types/layouts.py index a99214a43..54210da9b 100644 --- a/sharktank/sharktank/types/layouts.py +++ b/sharktank/sharktank/types/layouts.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/quantizers.py b/sharktank/sharktank/types/quantizers.py index 6dcc35140..21f1c89ec 100644 --- a/sharktank/sharktank/types/quantizers.py +++ b/sharktank/sharktank/types/quantizers.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/sharding.py b/sharktank/sharktank/types/sharding.py index 38dcd9829..a670cae33 100644 --- a/sharktank/sharktank/types/sharding.py +++ b/sharktank/sharktank/types/sharding.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/tensors.py b/sharktank/sharktank/types/tensors.py index b48fb1b52..eab7d5823 100644 --- a/sharktank/sharktank/types/tensors.py +++ b/sharktank/sharktank/types/tensors.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/types/theta.py b/sharktank/sharktank/types/theta.py index aa58353c6..3537726ec 100644 --- a/sharktank/sharktank/types/theta.py +++ b/sharktank/sharktank/types/theta.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/cli.py b/sharktank/sharktank/utils/cli.py index 0a90d114c..3a13b1d45 100644 --- a/sharktank/sharktank/utils/cli.py +++ b/sharktank/sharktank/utils/cli.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/debugging.py b/sharktank/sharktank/utils/debugging.py index c087b2542..1371732e2 100644 --- a/sharktank/sharktank/utils/debugging.py +++ b/sharktank/sharktank/utils/debugging.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/hf_datasets.py b/sharktank/sharktank/utils/hf_datasets.py index 3a04f9d11..e49623a8d 100644 --- a/sharktank/sharktank/utils/hf_datasets.py +++ b/sharktank/sharktank/utils/hf_datasets.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/io.py b/sharktank/sharktank/utils/io.py index 48ae9474d..62fd78f33 100644 --- a/sharktank/sharktank/utils/io.py +++ b/sharktank/sharktank/utils/io.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/logging.py b/sharktank/sharktank/utils/logging.py index 67868b6b4..977462d86 100644 --- a/sharktank/sharktank/utils/logging.py +++ b/sharktank/sharktank/utils/logging.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/math.py b/sharktank/sharktank/utils/math.py index 3056008ad..df47b5ae6 100644 --- a/sharktank/sharktank/utils/math.py +++ b/sharktank/sharktank/utils/math.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/patching.py b/sharktank/sharktank/utils/patching.py index a4b1c094a..aee70fd1e 100644 --- a/sharktank/sharktank/utils/patching.py +++ b/sharktank/sharktank/utils/patching.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/testing.py b/sharktank/sharktank/utils/testing.py index 0d0896fa4..8494c1d9b 100644 --- a/sharktank/sharktank/utils/testing.py +++ b/sharktank/sharktank/utils/testing.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/tokenizer.py b/sharktank/sharktank/utils/tokenizer.py index 833e8cfd9..29a57f958 100644 --- a/sharktank/sharktank/utils/tokenizer.py +++ b/sharktank/sharktank/utils/tokenizer.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/sharktank/utils/tree.py b/sharktank/sharktank/utils/tree.py index dcd63afe8..7736f2a90 100644 --- a/sharktank/sharktank/utils/tree.py +++ b/sharktank/sharktank/utils/tree.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/examples/main_test.py b/sharktank/tests/examples/main_test.py index befa62a73..fb43977df 100644 --- a/sharktank/tests/examples/main_test.py +++ b/sharktank/tests/examples/main_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/batch_matmul_transpose_b_test.py b/sharktank/tests/kernels/batch_matmul_transpose_b_test.py index 760e0055d..30cc2296c 100644 --- a/sharktank/tests/kernels/batch_matmul_transpose_b_test.py +++ b/sharktank/tests/kernels/batch_matmul_transpose_b_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/conv_2d_nchw_fchw_test.py b/sharktank/tests/kernels/conv_2d_nchw_fchw_test.py index bd4ff3b35..b03293523 100644 --- a/sharktank/tests/kernels/conv_2d_nchw_fchw_test.py +++ b/sharktank/tests/kernels/conv_2d_nchw_fchw_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/mmt_block_scaled_offset_q4_test.py b/sharktank/tests/kernels/mmt_block_scaled_offset_q4_test.py index 912f9b20c..d9fc7370a 100644 --- a/sharktank/tests/kernels/mmt_block_scaled_offset_q4_test.py +++ b/sharktank/tests/kernels/mmt_block_scaled_offset_q4_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/mmt_block_scaled_q8_test.py b/sharktank/tests/kernels/mmt_block_scaled_q8_test.py index 469c048a2..f3fdf2ed9 100644 --- a/sharktank/tests/kernels/mmt_block_scaled_q8_test.py +++ b/sharktank/tests/kernels/mmt_block_scaled_q8_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/mmt_super_block_scaled_offset_q4_test.py b/sharktank/tests/kernels/mmt_super_block_scaled_offset_q4_test.py index 328806508..41c04106d 100644 --- a/sharktank/tests/kernels/mmt_super_block_scaled_offset_q4_test.py +++ b/sharktank/tests/kernels/mmt_super_block_scaled_offset_q4_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/mmtfp_test.py b/sharktank/tests/kernels/mmtfp_test.py index 69b729b81..281498f90 100644 --- a/sharktank/tests/kernels/mmtfp_test.py +++ b/sharktank/tests/kernels/mmtfp_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/kernels/pooling_nchw_sum_test.py b/sharktank/tests/kernels/pooling_nchw_sum_test.py index 753b38119..5c4e8ac0a 100644 --- a/sharktank/tests/kernels/pooling_nchw_sum_test.py +++ b/sharktank/tests/kernels/pooling_nchw_sum_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/layers/linear_test.py b/sharktank/tests/layers/linear_test.py index 8db29b26b..e2d038f72 100644 --- a/sharktank/tests/layers/linear_test.py +++ b/sharktank/tests/layers/linear_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/llama/attention_test.py b/sharktank/tests/models/llama/attention_test.py index acc556f08..cf10c856f 100644 --- a/sharktank/tests/models/llama/attention_test.py +++ b/sharktank/tests/models/llama/attention_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/llama/kv_cache_test.py b/sharktank/tests/models/llama/kv_cache_test.py index 3953b951b..9d36db2e2 100644 --- a/sharktank/tests/models/llama/kv_cache_test.py +++ b/sharktank/tests/models/llama/kv_cache_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/punet/conftest.py b/sharktank/tests/models/punet/conftest.py index 1c88ceb31..55364c4ef 100644 --- a/sharktank/tests/models/punet/conftest.py +++ b/sharktank/tests/models/punet/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/punet/resnet_test.py b/sharktank/tests/models/punet/resnet_test.py index fc5762f7d..6ee696bbc 100644 --- a/sharktank/tests/models/punet/resnet_test.py +++ b/sharktank/tests/models/punet/resnet_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/punet/sharded_resnet_block_with_iree_test.py b/sharktank/tests/models/punet/sharded_resnet_block_with_iree_test.py index ee3c42926..e2b602a7c 100644 --- a/sharktank/tests/models/punet/sharded_resnet_block_with_iree_test.py +++ b/sharktank/tests/models/punet/sharded_resnet_block_with_iree_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/models/punet/up_down_block_test.py b/sharktank/tests/models/punet/up_down_block_test.py index 0013d85a5..acfcb35ab 100644 --- a/sharktank/tests/models/punet/up_down_block_test.py +++ b/sharktank/tests/models/punet/up_down_block_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/ops/ops_test.py b/sharktank/tests/ops/ops_test.py index 24a5f91b1..b282d03fe 100644 --- a/sharktank/tests/ops/ops_test.py +++ b/sharktank/tests/ops/ops_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/ops/qconv_test.py b/sharktank/tests/ops/qconv_test.py index 1c00a6c21..4440202eb 100644 --- a/sharktank/tests/ops/qconv_test.py +++ b/sharktank/tests/ops/qconv_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/ops/sharded_test.py b/sharktank/tests/ops/sharded_test.py index 34e5ebca7..39078c5b9 100644 --- a/sharktank/tests/ops/sharded_test.py +++ b/sharktank/tests/ops/sharded_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/transforms/dataset_transforms_test.py b/sharktank/tests/transforms/dataset_transforms_test.py index bb10bdeb1..4a57b6229 100644 --- a/sharktank/tests/transforms/dataset_transforms_test.py +++ b/sharktank/tests/transforms/dataset_transforms_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/types/dataset_test.py b/sharktank/tests/types/dataset_test.py index 99d176c5b..1164fdbcf 100644 --- a/sharktank/tests/types/dataset_test.py +++ b/sharktank/tests/types/dataset_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/types/layout_utils_test.py b/sharktank/tests/types/layout_utils_test.py index d468165e7..88194555e 100644 --- a/sharktank/tests/types/layout_utils_test.py +++ b/sharktank/tests/types/layout_utils_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/types/layouts_test.py b/sharktank/tests/types/layouts_test.py index 63fa75f9b..2be2990bf 100644 --- a/sharktank/tests/types/layouts_test.py +++ b/sharktank/tests/types/layouts_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/types/quantizers_test.py b/sharktank/tests/types/quantizers_test.py index 705808109..787725e88 100644 --- a/sharktank/tests/types/quantizers_test.py +++ b/sharktank/tests/types/quantizers_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/sharktank/tests/types/tensors_test.py b/sharktank/tests/types/tensors_test.py index 9bebcf14e..8ee839b6e 100644 --- a/sharktank/tests/types/tensors_test.py +++ b/sharktank/tests/types/tensors_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/setup.py b/shortfin/setup.py index 3dfb8aa8a..9ce1aa822 100644 --- a/shortfin/setup.py +++ b/shortfin/setup.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/framework/logging.py b/shortfin/shortfin/framework/logging.py index 5fae8aef3..4843ee358 100644 --- a/shortfin/shortfin/framework/logging.py +++ b/shortfin/shortfin/framework/logging.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/framework/session.py b/shortfin/shortfin/framework/session.py index 4a87e74c2..c28b81f5b 100644 --- a/shortfin/shortfin/framework/session.py +++ b/shortfin/shortfin/framework/session.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/api/rest_server.py b/shortfin/shortfin/llm/api/rest_server.py index a6fd28b57..33810428b 100644 --- a/shortfin/shortfin/llm/api/rest_server.py +++ b/shortfin/shortfin/llm/api/rest_server.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/attn_block_cache.py b/shortfin/shortfin/llm/attn_block_cache.py index 87222bb85..a9443a1e3 100644 --- a/shortfin/shortfin/llm/attn_block_cache.py +++ b/shortfin/shortfin/llm/attn_block_cache.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/config.py b/shortfin/shortfin/llm/config.py index 9a389fbfe..df5db5f8f 100644 --- a/shortfin/shortfin/llm/config.py +++ b/shortfin/shortfin/llm/config.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/impl/service_v1.py b/shortfin/shortfin/llm/impl/service_v1.py index 93f11d1d0..f3fd80a93 100644 --- a/shortfin/shortfin/llm/impl/service_v1.py +++ b/shortfin/shortfin/llm/impl/service_v1.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/impl/service_v1_cli.py b/shortfin/shortfin/llm/impl/service_v1_cli.py index 10f8abb42..9fc55b5ac 100644 --- a/shortfin/shortfin/llm/impl/service_v1_cli.py +++ b/shortfin/shortfin/llm/impl/service_v1_cli.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/service.py b/shortfin/shortfin/llm/service.py index 09ef135f2..c5d4ffb44 100644 --- a/shortfin/shortfin/llm/service.py +++ b/shortfin/shortfin/llm/service.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/shortfin/llm/testing/fake_v1_module.py b/shortfin/shortfin/llm/testing/fake_v1_module.py index 381304836..a36ebe667 100644 --- a/shortfin/shortfin/llm/testing/fake_v1_module.py +++ b/shortfin/shortfin/llm/testing/fake_v1_module.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/tests/framework/device_session_test.py b/shortfin/tests/framework/device_session_test.py index 35b82fb37..7b4916eb4 100644 --- a/shortfin/tests/framework/device_session_test.py +++ b/shortfin/tests/framework/device_session_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/tests/llm/api_server_test.py b/shortfin/tests/llm/api_server_test.py index 8bcd7c71a..fe4153dae 100644 --- a/shortfin/tests/llm/api_server_test.py +++ b/shortfin/tests/llm/api_server_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/shortfin/tests/llm/service_v1_test.py b/shortfin/tests/llm/service_v1_test.py index ad0db0a8d..56472ecc2 100644 --- a/shortfin/tests/llm/service_v1_test.py +++ b/shortfin/tests/llm/service_v1_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/tuner/candidate_gen.py b/tuner/candidate_gen.py index 8a8315afb..5a878e072 100755 --- a/tuner/candidate_gen.py +++ b/tuner/candidate_gen.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. diff --git a/tuner/candidate_gen_test.py b/tuner/candidate_gen_test.py index 52b01869b..ee0a32c66 100644 --- a/tuner/candidate_gen_test.py +++ b/tuner/candidate_gen_test.py @@ -1,4 +1,4 @@ -# Copyright 2024 Advanced Micro Devices, Inc +# Copyright 2024 Advanced Micro Devices, Inc. # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information.