From 101f55c686de01fbbbc63cd2f10019031e9f37a0 Mon Sep 17 00:00:00 2001 From: Ben Vanik Date: Wed, 18 Dec 2024 10:01:00 -0800 Subject: [PATCH] Adding fd-based file handle support to the HAL. (#19514) This allows multi-threaded unbuffered IO via pread/pwrite (with an emulation on Windows). --- runtime/src/iree/hal/utils/BUILD.bazel | 2 + runtime/src/iree/hal/utils/CMakeLists.txt | 2 + runtime/src/iree/hal/utils/fd_file.c | 384 +++++++++++++++++++++ runtime/src/iree/hal/utils/fd_file.h | 34 ++ runtime/src/iree/hal/utils/file_registry.c | 5 + runtime/src/iree/io/file_handle.c | 35 ++ runtime/src/iree/io/file_handle.h | 7 +- 7 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 runtime/src/iree/hal/utils/fd_file.c create mode 100644 runtime/src/iree/hal/utils/fd_file.h diff --git a/runtime/src/iree/hal/utils/BUILD.bazel b/runtime/src/iree/hal/utils/BUILD.bazel index 76d710e7dab0..61cd73d3a0ae 100644 --- a/runtime/src/iree/hal/utils/BUILD.bazel +++ b/runtime/src/iree/hal/utils/BUILD.bazel @@ -108,10 +108,12 @@ iree_runtime_cc_library( iree_runtime_cc_library( name = "files", srcs = [ + "fd_file.c", "file_registry.c", "memory_file.c", ], hdrs = [ + "fd_file.h", "file_registry.h", "memory_file.h", ], diff --git a/runtime/src/iree/hal/utils/CMakeLists.txt b/runtime/src/iree/hal/utils/CMakeLists.txt index b4e389e56eb9..e47c9ab1c4c3 100644 --- a/runtime/src/iree/hal/utils/CMakeLists.txt +++ b/runtime/src/iree/hal/utils/CMakeLists.txt @@ -130,9 +130,11 @@ iree_cc_library( NAME files HDRS + "fd_file.h" "file_registry.h" "memory_file.h" SRCS + "fd_file.c" "file_registry.c" "memory_file.c" DEPS diff --git a/runtime/src/iree/hal/utils/fd_file.c b/runtime/src/iree/hal/utils/fd_file.c new file mode 100644 index 000000000000..3a7fb60f0802 --- /dev/null +++ b/runtime/src/iree/hal/utils/fd_file.c @@ -0,0 +1,384 @@ +// Copyright 2024 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/hal/utils/fd_file.h" + +//===----------------------------------------------------------------------===// +// Platform Support +//===----------------------------------------------------------------------===// + +#if IREE_FILE_IO_ENABLE + +#define _GNU_SOURCE + +#include +#include +#include + +#if defined(IREE_PLATFORM_WINDOWS) +#include +#else +#include +#endif // IREE_PLATFORM_WINDOWS + +#if defined(IREE_PLATFORM_WINDOWS) + +// Returns the allowed access and length in bytes of the file descriptor. +// Returns 0 if the file descriptor has no length (a /proc stream, etc). +static iree_status_t iree_hal_platform_fd_stat( + int fd, iree_hal_memory_access_t* out_allowed_access, + uint64_t* out_length) { + IREE_ASSERT_ARGUMENT(out_allowed_access); + IREE_ASSERT_ARGUMENT(out_length); + *out_allowed_access = IREE_HAL_MEMORY_ACCESS_NONE; + *out_length = 0; + + struct _stat64 buffer = {0}; + if (_fstat64(fd, &buffer) == -1) { + return iree_make_status(iree_status_code_from_errno(errno), + "unable to stat file descriptor length"); + } + + *out_allowed_access = + ((buffer.st_mode & _S_IREAD) ? IREE_HAL_MEMORY_ACCESS_READ : 0) | + ((buffer.st_mode & _S_IWRITE) ? IREE_HAL_MEMORY_ACCESS_WRITE : 0); + *out_length = (uint64_t)buffer.st_size; + return iree_ok_status(); +} + +static iree_status_t iree_hal_platform_fd_pread( + int fd, void* buffer, iree_host_size_t count, uint64_t offset, + iree_host_size_t* out_bytes_read) { + IREE_ASSERT_ARGUMENT(out_bytes_read); + *out_bytes_read = 0; + + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "file descriptor is not backed by a valid Win32 HANDLE"); + } + + DWORD bytes_read = 0; + OVERLAPPED overlapped = {0}; + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFFu); + overlapped.OffsetHigh = (DWORD)((offset >> 32) & 0xFFFFFFFFu); + if (!ReadFile(handle, buffer, (DWORD)count, &bytes_read, &overlapped)) { + return iree_make_status(iree_status_code_from_win32_error(GetLastError()), + "failed to read requested buffer length"); + } + + *out_bytes_read = (iree_host_size_t)bytes_read; + return iree_ok_status(); +} + +static iree_status_t iree_hal_platform_fd_pwrite( + int fd, const void* buffer, iree_host_size_t count, uint64_t offset, + iree_host_size_t* out_bytes_written) { + IREE_ASSERT_ARGUMENT(out_bytes_written); + *out_bytes_written = 0; + + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "file descriptor is not backed by a valid Win32 HANDLE"); + } + + DWORD bytes_written = 0; + OVERLAPPED overlapped = {0}; + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFFu); + overlapped.OffsetHigh = (DWORD)((offset >> 32) & 0xFFFFFFFFu); + if (!WriteFile(handle, buffer, (DWORD)count, &bytes_written, &overlapped)) { + return iree_make_status(iree_status_code_from_win32_error(GetLastError()), + "failed to write requested buffer length"); + } + + *out_bytes_written = (iree_host_size_t)bytes_written; + return iree_ok_status(); +} + +#else + +// Returns the allowed access and length in bytes of the file descriptor. +// Returns 0 if the file descriptor has no length (a /proc stream, etc). +static iree_status_t iree_hal_platform_fd_stat( + int fd, iree_hal_memory_access_t* out_allowed_access, + uint64_t* out_length) { + IREE_ASSERT_ARGUMENT(out_allowed_access); + IREE_ASSERT_ARGUMENT(out_length); + *out_allowed_access = IREE_HAL_MEMORY_ACCESS_NONE; + *out_length = 0; + + struct stat buffer = {0}; + if (fstat(fd, &buffer) == -1) { + return iree_make_status(iree_status_code_from_errno(errno), + "unable to stat file descriptor length"); + } + + *out_allowed_access = + ((buffer.st_mode & S_IRUSR) ? IREE_HAL_MEMORY_ACCESS_READ : 0) | + ((buffer.st_mode & S_IWUSR) ? IREE_HAL_MEMORY_ACCESS_WRITE : 0); + *out_length = (uint64_t)buffer.st_size; + return iree_ok_status(); +} + +static iree_status_t iree_hal_platform_fd_pread( + int fd, void* buffer, iree_host_size_t count, uint64_t offset, + iree_host_size_t* out_bytes_read) { + IREE_ASSERT_ARGUMENT(out_bytes_read); + *out_bytes_read = 0; + ssize_t bytes_read = pread(fd, buffer, (size_t)count, (off_t)offset); + if (bytes_read > 0) { + *out_bytes_read = (iree_host_size_t)bytes_read; + return iree_ok_status(); + } else if (bytes_read == 0) { + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, + "end of file hit during read"); + } else { + return iree_make_status(iree_status_code_from_errno(errno), + "failed to read requested buffer length"); + } +} + +static iree_status_t iree_hal_platform_fd_pwrite( + int fd, const void* buffer, iree_host_size_t count, uint64_t offset, + iree_host_size_t* out_bytes_written) { + IREE_ASSERT_ARGUMENT(out_bytes_written); + *out_bytes_written = 0; + ssize_t bytes_written = pwrite(fd, buffer, (size_t)count, (off_t)offset); + if (bytes_written > 0) { + *out_bytes_written = (iree_host_size_t)bytes_written; + return iree_ok_status(); + } else if (bytes_written == 0) { + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, + "end of file hit during write"); + } else { + return iree_make_status(iree_status_code_from_errno(errno), + "failed to write requested buffer length"); + } +} + +#endif // IREE_PLATFORM_WINDOWS + +#endif // IREE_FILE_IO_ENABLE + +//===----------------------------------------------------------------------===// +// iree_hal_fd_file_t +//===----------------------------------------------------------------------===// + +#if IREE_FILE_IO_ENABLE + +typedef struct iree_hal_fd_file_t { + iree_hal_resource_t resource; + // Used to allocate this structure. + iree_allocator_t host_allocator; + // Allowed access bits. + iree_hal_memory_access_t access; + // Base file handle, retained. + iree_io_file_handle_t* handle; + // File descriptor, unretained (the handle retains it). + // Note that this descriptor may be shared with multiple threads and all + // operations we perform against it must be stateless. + int fd; + // Total file (stream) length in bytes as queried on creation. + uint64_t length; +} iree_hal_fd_file_t; + +static const iree_hal_file_vtable_t iree_hal_fd_file_vtable; + +static iree_hal_fd_file_t* iree_hal_fd_file_cast( + iree_hal_file_t* IREE_RESTRICT base_value) { + return (iree_hal_fd_file_t*)base_value; +} + +IREE_API_EXPORT iree_status_t iree_hal_fd_file_from_handle( + iree_hal_memory_access_t access, iree_io_file_handle_t* handle, + iree_allocator_t host_allocator, iree_hal_file_t** out_file) { + IREE_ASSERT_ARGUMENT(out_file); + *out_file = NULL; + + // For now we only support posix file descriptors but could support other + // handle types so long as they are compatible with pread/pwrite. + iree_io_file_handle_primitive_t primitive = + iree_io_file_handle_primitive(handle); + if (primitive.type != IREE_IO_FILE_HANDLE_TYPE_FD) { + return iree_make_status(IREE_STATUS_UNIMPLEMENTED, + "support for creating non-fd files not supported"); + } + const int fd = primitive.value.fd; + + IREE_TRACE_ZONE_BEGIN(z0); + + // Query the file length. This also acts as a quick check that the file + // descriptor is accessible. + iree_hal_memory_access_t allowed_access = IREE_HAL_MEMORY_ACCESS_NONE; + uint64_t length = 0; + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_hal_platform_fd_stat(fd, &allowed_access, &length)); + + // Verify that the requested access can be satisfied. + if (iree_all_bits_set(access, IREE_HAL_MEMORY_ACCESS_READ) && + !iree_all_bits_set(allowed_access, IREE_HAL_MEMORY_ACCESS_READ)) { + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, + iree_make_status( + IREE_STATUS_PERMISSION_DENIED, + "read access requested on a file descriptor that is not readable")); + } else if (iree_all_bits_set(access, IREE_HAL_MEMORY_ACCESS_WRITE) && + !iree_all_bits_set(allowed_access, IREE_HAL_MEMORY_ACCESS_WRITE)) { + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_make_status(IREE_STATUS_PERMISSION_DENIED, + "write access requested on a file descriptor that " + "is not writable")); + } + + // Allocate object that retains the underlying file handle and our opened + // descriptor. + iree_hal_fd_file_t* file = NULL; + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_allocator_malloc(host_allocator, sizeof(*file), (void**)&file)); + iree_hal_resource_initialize(&iree_hal_fd_file_vtable, &file->resource); + file->host_allocator = host_allocator; + file->access = access; + file->handle = handle; + iree_io_file_handle_retain(file->handle); + file->fd = fd; + file->length = length; + + *out_file = (iree_hal_file_t*)file; + IREE_TRACE_ZONE_END(z0); + return iree_ok_status(); +} + +static void iree_hal_fd_file_destroy(iree_hal_file_t* IREE_RESTRICT base_file) { + iree_hal_fd_file_t* file = iree_hal_fd_file_cast(base_file); + iree_allocator_t host_allocator = file->host_allocator; + IREE_TRACE_ZONE_BEGIN(z0); + + iree_io_file_handle_release(file->handle); + + iree_allocator_free(host_allocator, file); + + IREE_TRACE_ZONE_END(z0); +} + +static iree_hal_memory_access_t iree_hal_fd_file_allowed_access( + iree_hal_file_t* base_file) { + iree_hal_fd_file_t* file = iree_hal_fd_file_cast(base_file); + return file->access; +} + +static uint64_t iree_hal_fd_file_length(iree_hal_file_t* base_file) { + iree_hal_fd_file_t* file = iree_hal_fd_file_cast(base_file); + return file->length; +} + +static iree_hal_buffer_t* iree_hal_fd_file_storage_buffer( + iree_hal_file_t* base_file) { + // We could map files if we wanted to provide this interface but today leave + // that up to users (they can pass in HOST_ALLOCATION file handles to import). + return NULL; +} + +static bool iree_hal_fd_file_supports_synchronous_io( + iree_hal_file_t* base_file) { + // Host files always support synchronous IO. + return true; +} + +static iree_status_t iree_hal_fd_file_read(iree_hal_file_t* base_file, + uint64_t file_offset, + iree_hal_buffer_t* buffer, + iree_device_size_t buffer_offset, + iree_device_size_t length) { + if (length == 0) return iree_ok_status(); + iree_hal_fd_file_t* file = iree_hal_fd_file_cast(base_file); + + iree_hal_buffer_mapping_t mapping = {{0}}; + IREE_RETURN_IF_ERROR(iree_hal_buffer_map_range( + buffer, IREE_HAL_MAPPING_MODE_SCOPED, + IREE_HAL_MEMORY_ACCESS_DISCARD_WRITE, buffer_offset, length, &mapping)); + + iree_status_t status = iree_ok_status(); + uint8_t* buffer_ptr = mapping.contents.data; + iree_host_size_t bytes_remaining = mapping.contents.data_length; + while (iree_status_is_ok(status) && bytes_remaining > 0) { + const iree_host_size_t bytes_requested = iree_min(bytes_remaining, INT_MAX); + iree_host_size_t bytes_read = 0; + status = iree_hal_platform_fd_pread(file->fd, buffer_ptr, bytes_requested, + file_offset, &bytes_read); + file_offset += bytes_read; + buffer_ptr += bytes_read; + bytes_remaining -= bytes_read; + } + + if (iree_status_is_ok(status) && + !iree_all_bits_set(iree_hal_buffer_memory_type(buffer), + IREE_HAL_MEMORY_TYPE_HOST_COHERENT)) { + status = + iree_hal_buffer_mapping_flush_range(&mapping, buffer_offset, length); + } + + return iree_status_join(status, iree_hal_buffer_unmap_range(&mapping)); +} + +static iree_status_t iree_hal_fd_file_write(iree_hal_file_t* base_file, + uint64_t file_offset, + iree_hal_buffer_t* buffer, + iree_device_size_t buffer_offset, + iree_device_size_t length) { + if (length == 0) return iree_ok_status(); + iree_hal_fd_file_t* file = iree_hal_fd_file_cast(base_file); + + iree_hal_buffer_mapping_t mapping = {{0}}; + IREE_RETURN_IF_ERROR(iree_hal_buffer_map_range( + buffer, IREE_HAL_MAPPING_MODE_SCOPED, IREE_HAL_MEMORY_ACCESS_READ, + buffer_offset, length, &mapping)); + + iree_status_t status = iree_ok_status(); + if (!iree_all_bits_set(iree_hal_buffer_memory_type(buffer), + IREE_HAL_MEMORY_TYPE_HOST_COHERENT)) { + status = iree_hal_buffer_mapping_invalidate_range(&mapping, buffer_offset, + length); + } + + const uint8_t* buffer_ptr = mapping.contents.data; + iree_host_size_t bytes_remaining = mapping.contents.data_length; + while (iree_status_is_ok(status) && bytes_remaining > 0) { + const iree_host_size_t bytes_requested = iree_min(bytes_remaining, INT_MAX); + iree_host_size_t bytes_written = 0; + status = iree_hal_platform_fd_pwrite(file->fd, buffer_ptr, bytes_requested, + file_offset, &bytes_written); + file_offset += bytes_written; + buffer_ptr += bytes_written; + bytes_remaining -= bytes_written; + } + + return iree_status_join(status, iree_hal_buffer_unmap_range(&mapping)); +} + +static const iree_hal_file_vtable_t iree_hal_fd_file_vtable = { + .destroy = iree_hal_fd_file_destroy, + .allowed_access = iree_hal_fd_file_allowed_access, + .length = iree_hal_fd_file_length, + .storage_buffer = iree_hal_fd_file_storage_buffer, + .supports_synchronous_io = iree_hal_fd_file_supports_synchronous_io, + .read = iree_hal_fd_file_read, + .write = iree_hal_fd_file_write, +}; + +#else + +IREE_API_EXPORT iree_status_t iree_hal_fd_file_from_handle( + iree_hal_memory_access_t access, iree_io_file_handle_t* handle, + iree_allocator_t host_allocator, iree_hal_file_t** out_file) { + return iree_make_status(IREE_STATUS_UNAVAILABLE, + "file support has been compiled out of this binary; " + "set IREE_FILE_IO_ENABLE=1 to include it"); +} + +#endif // IREE_FILE_IO_ENABLE diff --git a/runtime/src/iree/hal/utils/fd_file.h b/runtime/src/iree/hal/utils/fd_file.h new file mode 100644 index 000000000000..f6f8f371d4c8 --- /dev/null +++ b/runtime/src/iree/hal/utils/fd_file.h @@ -0,0 +1,34 @@ +// Copyright 2024 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_HAL_UTILS_FD_FILE_H_ +#define IREE_HAL_UTILS_FD_FILE_H_ + +#include "iree/base/api.h" +#include "iree/hal/api.h" +#include "iree/io/file_handle.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +//===----------------------------------------------------------------------===// +// iree_hal_fd_file_t +//===----------------------------------------------------------------------===// + +// Creates a file backed by |handle| on disk. +// Only supports file handles of IREE_IO_FILE_HANDLE_TYPE_FD. +// File handles are stateless and each host file opened from one may see +// different versions of the file depending on the platform and file type. +IREE_API_EXPORT iree_status_t iree_hal_fd_file_from_handle( + iree_hal_memory_access_t access, iree_io_file_handle_t* handle, + iree_allocator_t host_allocator, iree_hal_file_t** out_file); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // IREE_HAL_UTILS_FD_FILE_H_ diff --git a/runtime/src/iree/hal/utils/file_registry.c b/runtime/src/iree/hal/utils/file_registry.c index 3926c394b09f..8e80e9d82c27 100644 --- a/runtime/src/iree/hal/utils/file_registry.c +++ b/runtime/src/iree/hal/utils/file_registry.c @@ -6,6 +6,7 @@ #include "iree/hal/utils/file_registry.h" +#include "iree/hal/utils/fd_file.h" #include "iree/hal/utils/memory_file.h" IREE_API_EXPORT iree_status_t iree_hal_file_from_handle( @@ -25,6 +26,10 @@ IREE_API_EXPORT iree_status_t iree_hal_file_from_handle( iree_hal_memory_file_wrap(device_allocator, queue_affinity, access, handle, host_allocator, out_file); break; + case IREE_IO_FILE_HANDLE_TYPE_FD: + status = iree_hal_fd_file_from_handle(access, handle, host_allocator, + out_file); + break; default: status = iree_make_status( IREE_STATUS_UNIMPLEMENTED, diff --git a/runtime/src/iree/io/file_handle.c b/runtime/src/iree/io/file_handle.c index 2eb7e19940e7..dea7fcaee8b1 100644 --- a/runtime/src/iree/io/file_handle.c +++ b/runtime/src/iree/io/file_handle.c @@ -9,6 +9,18 @@ #include "iree/base/internal/atomics.h" #include "iree/io/memory_stream.h" +#if IREE_FILE_IO_ENABLE +#if defined(IREE_PLATFORM_WINDOWS) + +#include // _commit + +#else + +#include // fsync + +#endif // IREE_PLATFORM_WINDOWS +#endif // IREE_FILE_IO_ENABLE + //===----------------------------------------------------------------------===// // iree_io_file_handle_t //===----------------------------------------------------------------------===// @@ -101,6 +113,25 @@ iree_io_file_handle_primitive(const iree_io_file_handle_t* handle) { return handle->primitive; } +static iree_status_t iree_io_platform_fd_flush(int fd) { +#if IREE_FILE_IO_ENABLE + +#if defined(IREE_PLATFORM_WINDOWS) + int ret = _commit(fd); +#else + int ret = fsync(fd); +#endif // IREE_PLATFORM_WINDOWS + return ret != -1 ? iree_ok_status() + : iree_make_status(iree_status_code_from_errno(errno), + "unable to sync file writes"); + +#else + return iree_make_status(IREE_STATUS_UNAVAILABLE, + "file support has been compiled out of this binary; " + "set IREE_FILE_IO_ENABLE=1 to include it"); +#endif // IREE_FILE_IO_ENABLE +} + IREE_API_EXPORT iree_status_t iree_io_file_handle_flush(iree_io_file_handle_t* handle) { IREE_ASSERT_ARGUMENT(handle); @@ -111,6 +142,10 @@ iree_io_file_handle_flush(iree_io_file_handle_t* handle) { // No-op (though we could flush when known mapped). break; } + case IREE_IO_FILE_HANDLE_TYPE_FD: { + status = iree_io_platform_fd_flush(handle->primitive.value.fd); + break; + } default: { status = iree_make_status(IREE_STATUS_UNIMPLEMENTED, "flush not supported on handle type %d", diff --git a/runtime/src/iree/io/file_handle.h b/runtime/src/iree/io/file_handle.h index 6831cbe6c3e1..6d46ad559d4c 100644 --- a/runtime/src/iree/io/file_handle.h +++ b/runtime/src/iree/io/file_handle.h @@ -46,7 +46,10 @@ typedef enum iree_io_file_handle_type_e { // as long as the file handle referencing it. IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION = 0u, - // TODO(benvanik): file descriptor, FILE*, HANDLE, etc. + // Platform file descriptor (fd). + IREE_IO_FILE_HANDLE_TYPE_FD, + + // TODO(benvanik): FILE*, HANDLE, etc. } iree_io_file_handle_type_t; // A platform handle to a file primitive. @@ -55,6 +58,8 @@ typedef enum iree_io_file_handle_type_e { typedef union iree_io_file_handle_primitive_value_t { // IREE_IO_FILE_HANDLE_TYPE_HOST_ALLOCATION iree_byte_span_t host_allocation; + // IREE_IO_FILE_HANDLE_TYPE_FD + int fd; } iree_io_file_handle_primitive_value_t; // A (type, value) pair describing a system file primitive handle.