From 652bc76c1553df35c22d95c0f794ca6f272779a6 Mon Sep 17 00:00:00 2001 From: Hongzhen Luo Date: Tue, 4 Jun 2024 10:26:16 +0800 Subject: [PATCH] Introduce tarerofs I/O API Previously, erofs-utils was integrated into overlaybd by using the `mkfs.erofs` executable file. It's not optimial since raw data needs to be dumped first and output data needs to be write into overlaybd then. This commit introduces the I/O API in tarerofs, that the data stream of the: - output image - input tar file can be controlled by the I/O manager. This also adds block read/write cache support for IFile to improve the efficiency of making file system images. Signed-off-by: Hongzhen Luo --- src/overlaybd/tar/CMakeLists.txt | 3 + src/overlaybd/tar/erofs/CMakeLists.txt | 43 ++ src/overlaybd/tar/erofs/liberofs.cpp | 687 +++++++++++++++++++++++++ src/overlaybd/tar/erofs/liberofs.h | 17 + src/overlaybd/tar/tarerofs.cpp | 139 ----- src/overlaybd/tar/tarerofs.h | 37 -- src/tools/turboOCI-apply.cpp | 8 +- 7 files changed, 753 insertions(+), 181 deletions(-) create mode 100644 src/overlaybd/tar/erofs/CMakeLists.txt create mode 100644 src/overlaybd/tar/erofs/liberofs.cpp create mode 100644 src/overlaybd/tar/erofs/liberofs.h delete mode 100644 src/overlaybd/tar/tarerofs.cpp delete mode 100644 src/overlaybd/tar/tarerofs.h diff --git a/src/overlaybd/tar/CMakeLists.txt b/src/overlaybd/tar/CMakeLists.txt index fe06bc11..a3eb8ed0 100644 --- a/src/overlaybd/tar/CMakeLists.txt +++ b/src/overlaybd/tar/CMakeLists.txt @@ -8,3 +8,6 @@ target_include_directories(tar_lib PUBLIC if(BUILD_TESTING) add_subdirectory(test) endif() + +add_subdirectory(erofs) +target_link_libraries(tar_lib PRIVATE erofs_lib) \ No newline at end of file diff --git a/src/overlaybd/tar/erofs/CMakeLists.txt b/src/overlaybd/tar/erofs/CMakeLists.txt new file mode 100644 index 00000000..4435cb1d --- /dev/null +++ b/src/overlaybd/tar/erofs/CMakeLists.txt @@ -0,0 +1,43 @@ +include(FetchContent) + +FetchContent_Declare( + erofs-utils + GIT_REPOSITORY https://gitee.com/anolis/erofs-utils.git + GIT_TAG overlaybd-dev-2 +) + +FetchContent_MakeAvailable(erofs-utils) + +execute_process( + COMMAND ./autogen.sh + WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} +) +execute_process( + COMMAND ./configure --disable-lz4 --without-liblzma --without-libzstd + WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} +) +execute_process( + COMMAND make + WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} +) + +set(EROFS_LIB_INCLUDE_DIR "${erofs-utils_SOURCE_DIR}/include/" CACHE PATH "erofs-utils include path.") +set(EROFS_CONFIG_FILE "${erofs-utils_SOURCE_DIR}/config.h" CACHE PATH "erofs-utils config file.") +set(EROFS_LIB_STATIC "${erofs-utils_SOURCE_DIR}/lib/.libs/liberofs.a" CACHE PATH "erofs-utils static lib.") + +file(GLOB EROFS_SOURCE "*.cpp") + +add_library(erofs_lib STATIC ${EROFS_SOURCE}) + +target_include_directories(erofs_lib PRIVATE + ${PHOTON_INCLUDE_DIR} +) + +target_include_directories(erofs_lib PRIVATE + ${EROFS_LIB_INCLUDE_DIR} +) + +target_link_libraries(erofs_lib PRIVATE uuid) + +target_compile_options(erofs_lib PRIVATE "-include${EROFS_CONFIG_FILE}") +target_link_libraries(erofs_lib PRIVATE ${EROFS_LIB_STATIC}) \ No newline at end of file diff --git a/src/overlaybd/tar/erofs/liberofs.cpp b/src/overlaybd/tar/erofs/liberofs.cpp new file mode 100644 index 00000000..e21b8f1f --- /dev/null +++ b/src/overlaybd/tar/erofs/liberofs.cpp @@ -0,0 +1,687 @@ +#include "liberofs.h" +#include "erofs/tar.h" +#include "erofs/io.h" +#include "erofs/cache.h" +#include "erofs/block_list.h" +#include "erofs/inode.h" +#include "../../lsmt/file.h" +#include "../../lsmt/index.h" +#include +#include +#include + +#define SECTOR_SIZE 512ULL +#define SECTOR_BITS 9 +#define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) +#define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) +#define min(a, b) (a) < (b) ? (a) : (b) +#define MAP_FILE_NAME "upper.map" + +#define EROFS_UNIMPLEMENTED 1 + +struct liberofs_inmem_sector { + char data[SECTOR_SIZE]; +}; + +class ErofsCache { +public: + ErofsCache() {} + ~ErofsCache() {} + ssize_t write_sector(u64 addr, char *buf); + ssize_t read_sector(u64 addr, char *buf); + int flush(); +public: + photon::fs::IFile *file; + long unsigned int capacity; + std::mapcaches; + std::set dirty; +}; + +static struct erofs_vfops target_vfops; +static struct erofs_vfops source_vfops; +static ErofsCache erofs_cache; +static photon::fs::IFile *_target; +static photon::fs::IFile *_source; + +ssize_t ErofsCache::write_sector(u64 addr, char *buf) +{ + struct liberofs_inmem_sector *sector; + + if (addr & (SECTOR_SIZE-1)) { + LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); + return -1; + } + if (caches.find(addr) != caches.end()) { + sector = caches[addr]; + memcpy(sector->data, buf, SECTOR_SIZE); + dirty.insert(addr); + } else { + if (caches.size() == capacity) { + auto it = caches.begin(); + if (dirty.find(it->first) != dirty.end()) { + if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", it->first); + return -EIO; + } + dirty.erase(it->first); + } + sector = it->second; + caches.erase(it); + } else { + sector = (struct liberofs_inmem_sector*)malloc( + sizeof(struct liberofs_inmem_sector)); + if (!sector) + return -ENOMEM; + } + memcpy(sector->data, buf, SECTOR_SIZE); + caches[addr] = sector; + dirty.insert(addr); + } + return SECTOR_SIZE; +} + +ssize_t ErofsCache::read_sector(u64 addr, char *buf) +{ + struct liberofs_inmem_sector *sector; + + if (addr & (SECTOR_SIZE - 1)) { + LOG_ERROR("Invalid addr, should be aligned to SECTOR_SIZE."); + return -1; + } + + if (caches.find(addr) != caches.end()) { + sector = caches[addr]; + memcpy(buf, sector->data, SECTOR_SIZE); + } else { + if (caches.size() == capacity) { + auto it = caches.begin(); + if (dirty.find(it->first) != dirty.end()) { + sector = it->second; + if (file->pwrite(sector->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", it->first); + return -EIO; + } + dirty.erase(it->first); + } + sector = it->second; + caches.erase(it); + } else { + sector = (struct liberofs_inmem_sector*)malloc( + sizeof(struct liberofs_inmem_sector)); + if (!sector) + return -ENOMEM; + } + if (file->pread(sector->data, SECTOR_SIZE, addr) != SECTOR_SIZE) { + LOG_ERROR("Fail to read sector %lld", addr); + return -EIO; + } + caches[addr] = sector; + memcpy(buf, sector->data, SECTOR_SIZE); + } + return SECTOR_SIZE; +} + +int ErofsCache::flush() +{ + for (auto it = caches.begin(); it != caches.end();) { + if (dirty.find(it->first) != dirty.end()) { + if (file->pwrite(it->second->data, SECTOR_SIZE, it->first) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to flush sector %lld.\n", it->first); + return -1; + } + dirty.erase(it->first); + } + free(it->second); + it = caches.erase(it); + } + return dirty.size() != 0 || caches.size() != 0; +} + +/* + * Helper function for reading from the photon file, since + * the photon file requires reads to be 512-byte aligned. + */ +static ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, + ErofsCache *cache) +{ + + ssize_t read; + u64 start, end; + char extra_buf[SECTOR_SIZE]; + u64 i, j; + + start = round_down_blk(offset); + end = round_up_blk(offset + len); + read = 0; + + /* we use the extra_buf to store the first sector or last sector when: + * - start != offset + * or + * - end != offset + len + */ + if (start != offset || end != offset + len) { + i = start == offset ? start : start + SECTOR_SIZE; + j = end == offset + len ? end : end - SECTOR_SIZE; + + /* read the first sector */ + if (i != start) { + if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to read start sector."); + return -1; + } + memcpy((char*)buf, extra_buf + offset - start, + min(start + SECTOR_SIZE - offset,len)); + read += min(start + SECTOR_SIZE - offset,len); + } + + /* read the last sector and avoid re-reading the same sector as above */ + if (j != end && (i == start || end - start > SECTOR_SIZE)) { + if (cache->read_sector(end - SECTOR_SIZE, + extra_buf) != SECTOR_SIZE) + { + LOG_ERROR("Fail to read start sector."); + return -1; + } + memcpy((char*)buf + end - SECTOR_SIZE - offset, extra_buf, + len + offset + SECTOR_SIZE - end); + read += len + offset + SECTOR_SIZE - end; + } + + for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { + if (cache->read_sector(addr, + addr - offset + (char*)buf) != SECTOR_SIZE) + { + LOG_ERROR("Fail to read sector %lld in read_photo_file.\n", i); + return -1; + } + read += SECTOR_SIZE; + } + } else { + /* if read request is sector-aligned, we use the original buffer */ + for (u64 i = start; i < end; i += SECTOR_SIZE) { + if (cache->read_sector(i, (char*)buf + i - start) != SECTOR_SIZE) { + LOG_ERROR("Fail to read start sector."); + return -1; + } + read += SECTOR_SIZE; + } + } + + return read; +} + +/* + * Helper function for writing to a photon file. + */ + static ssize_t erofs_write_photon_file(const void *buf, u64 offset, + size_t len, ErofsCache *cache) + { + ssize_t write; + u64 start, end; + char extra_buf[SECTOR_SIZE]; + u64 i, j; + + start = round_down_blk(offset); + end = round_up_blk(offset + len); + write = 0; + + if (start != offset || end != offset + len) { + i = start == offset ? start : start + SECTOR_SIZE; + j = end == offset + len ? end : end - SECTOR_SIZE; + + if (i != start) { + if (cache->read_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to read sector %lld.\n", start); + return -1; + } + memcpy(extra_buf + offset - start, buf, + min(start + SECTOR_SIZE - offset, len)); + if (cache->write_sector(start, extra_buf) != SECTOR_SIZE) { + LOG_ERROR("Fail to write sector %lld\n", start); + return -1; + } + write += min(start + SECTOR_SIZE - offset, len); + } + + if (j != end && (i == start || end - start > SECTOR_SIZE)) { + if (cache->read_sector(end - SECTOR_SIZE, extra_buf) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to read sector %lld.", end - SECTOR_SIZE); + return -1; + } + memcpy(extra_buf, buf + end - SECTOR_SIZE - offset, + offset + len + SECTOR_SIZE - end); + if (cache->write_sector(end - SECTOR_SIZE, extra_buf) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", end - SECTOR_SIZE); + return -1; + } + write += offset + len + SECTOR_SIZE - end; + } + + for (u64 addr = i; addr < j; addr += SECTOR_SIZE) { + if (cache->write_sector(addr, (char*)buf + addr - offset) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", addr); + return -1; + } + write += SECTOR_SIZE; + } + } else { + for (u64 addr = start; addr < end; addr += SECTOR_SIZE) { + if (cache->write_sector(addr, (char *)buf + addr - start) + != SECTOR_SIZE) + { + LOG_ERROR("Fail to write sector %lld.", addr); + return -1; + } + write += SECTOR_SIZE; + } + } + + return write; +} + +/* I/O control for target */ +static ssize_t erofs_target_pread(struct erofs_vfile *vf, void *buf, u64 offset, + size_t len) +{ + if (erofs_read_photon_file(buf, offset, len, &erofs_cache) != (ssize_t)len) + return -1; + + return len; +} + +static ssize_t erofs_target_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len) +{ + if (!buf) + return -EINVAL; + + return erofs_write_photon_file(buf, offset, len, &erofs_cache); +} + +static int erofs_target_fsync(struct erofs_vfile *vf) +{ + return erofs_cache.flush(); +} + +static int erofs_target_fallocate(struct erofs_vfile *vf, u64 offset, + size_t len, bool pad) +{ + static const char zero[4096] = {0}; + ssize_t ret; + + while (len > 4096) { + ret = erofs_target_pwrite(vf, zero, offset, 4096); + if (ret) + return ret; + len -= 4096; + offset += 4096; + } + ret = erofs_target_pwrite(vf, zero, offset, len); + if (ret != (ssize_t)len) { + return -1; + } + return 0; +} + +static int erofs_target_ftruncate(struct erofs_vfile *vf, u64 length) +{ + return 0; +} + +static ssize_t erofs_target_read(struct erofs_vfile *vf, void *buf, size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +static off_t erofs_target_lseek(struct erofs_vfile *vf, u64 offset, int whence) +{ + return -EROFS_UNIMPLEMENTED; +} + +/* I/O control for source */ +static ssize_t erofs_source_pread(struct erofs_vfile *vf, void *buf, u64 offset, + size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +static ssize_t erofs_source_pwrite(struct erofs_vfile *vf, const void *buf, + u64 offset, size_t len) +{ + return -EROFS_UNIMPLEMENTED; +} + +static int erofs_source_fsync(struct erofs_vfile *vf) +{ + return -EROFS_UNIMPLEMENTED; +} + +static int erofs_source_fallocate(struct erofs_vfile *vf, + u64 offset, size_t len, bool pad) +{ + return -EROFS_UNIMPLEMENTED; +} + +static int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length) +{ + return EROFS_UNIMPLEMENTED; +} + +static ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf, + size_t bytes) +{ + u64 i = 0; + while (bytes) { + u64 len = bytes > INT_MAX ? INT_MAX : bytes; + u64 ret; + + ret = _source->read(buf + i, len); + if (ret < 1) { + if (ret == 0) + break; + else + return -1; + } + bytes -= ret; + i += ret; + } + return i; +} + +static off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, int whence) +{ + return _source->lseek(offset, whence); +} + +struct erofs_mkfs_cfg { + struct erofs_sb_info *sbi; + struct erofs_tarfile *erofstar; + bool incremental; + bool ovlfs_strip; +}; + +static int rebuild_src_count; + +int erofs_mkfs(struct erofs_mkfs_cfg *cfg) +{ + int err; + struct erofs_tarfile *erofstar; + struct erofs_sb_info *sbi; + struct erofs_buffer_head *sb_bh; + struct erofs_inode *root; + erofs_blk_t nblocks; + + erofstar = cfg->erofstar; + sbi = cfg->sbi; + if (!erofstar || !sbi) + return -EINVAL; + + if (!erofstar->mapfile) + return -EINVAL; + + err = erofs_blocklist_open(erofstar->mapfile, true); + if (err) { + LOG_ERROR("[erofs] Fail to open erofs blocklist."); + return -EINVAL; + } + + if (!erofstar->rvsp_mode) { + LOG_ERROR("[erofs] Must be in RVSP mode."); + return -EINVAL; + } + + if (!cfg->incremental) { + sb_bh = erofs_reserve_sb(sbi); + if (IS_ERR(sb_bh)) { + LOG_ERROR("[erofs] Fail to reseve space for superblock."); + err = PTR_ERR(sb_bh); + goto exit; + } + //erofs_uuid_generate(sbi->uuid); + } else { + err = erofs_read_superblock(sbi); + if (err) { + LOG_ERROR("[erofs] Fail to read superblock."); + goto exit; + } + erofs_buffer_init(sbi, sbi->primarydevice_blocks); + sb_bh = NULL; + } + + erofs_inode_manager_init(); + + root = erofs_rebuild_make_root(sbi); + if (IS_ERR(root)) { + LOG_ERROR("[erofs] Fail to alloc root inode."); + err = PTR_ERR(root); + goto exit; + } + + while (!(err = tarerofs_parse_tar(root, erofstar))); + if (err < 0) { + LOG_ERROR("[erofs] Fail to parse tar file."); + goto exit; + } + + err = erofs_rebuild_dump_tree(root, cfg->incremental, cfg->ovlfs_strip); + if (err < 0) { + LOG_ERROR("[erofs] Fail to dump tree."); + goto exit; + } + + err = erofs_bflush(NULL); + if (err) { + LOG_ERROR("[erofs] Bflush failed."); + goto exit; + } + + erofs_fixup_root_inode(root); + erofs_iput(root); + root = NULL; + + err = erofs_writesb(sbi, sb_bh, &nblocks); + if (err) { + LOG_ERROR("[erofs] Fail to writesb"); + goto exit; + } + + /* flush all remaining buffers */ + err = erofs_bflush(NULL); + if (err) + goto exit; + + err = erofs_dev_resize(sbi, nblocks); +exit: + if (root) + erofs_iput(root); + erofs_blocklist_close(); + return err; +} + +static int erofs_init_sbi(struct erofs_sb_info *sbi, photon::fs::IFile *fout, + struct erofs_vfops *ops, int blkbits) +{ + int err; + struct timeval t; + + sbi->blkszbits = (char)blkbits; + err = gettimeofday(&t, NULL); + if (err) + return err; + sbi->build_time = t.tv_sec; + sbi->build_time_nsec = t.tv_usec; + sbi->bdev.ops = ops; + fout->lseek(0, 0); + sbi->devsz = INT64_MAX; + + return 0; +} + +static int erofs_init_tar(struct erofs_tarfile *erofstar, + photon::fs::IFile *tar_file, struct erofs_vfops *ops) +{ + int err; + struct stat st; + + erofstar->global.xattrs = LIST_HEAD_INIT(erofstar->global.xattrs); + erofstar->mapfile = MAP_FILE_NAME; + erofstar->aufs = true; + erofstar->rvsp_mode = true; + erofstar->dev = rebuild_src_count + 1; + + erofstar->ios.feof = false; + erofstar->ios.tail = erofstar->ios.head = 0; + erofstar->ios.dumpfd = -1; + err = tar_file->fstat(&st); + if (err) { + LOG_ERROR("Fail to fstat tar file."); + return err; + } + erofstar->ios.sz = st.st_size; + erofstar->ios.bufsize = 16384; + do { + erofstar->ios.buffer = (char*)malloc(erofstar->ios.bufsize); + if (erofstar->ios.buffer) + break; + erofstar->ios.bufsize >>= 1; + } while (erofstar->ios.bufsize >= 1024); + + if (!erofstar->ios.buffer) + return -ENOMEM; + + erofstar->ios.vf.ops = ops; + + return 0; +} + +static int erofs_write_map_file(photon::fs::IFile *fout, uint64_t blksz) +{ + FILE *fp; + uint64_t blkaddr, toff; + uint32_t nblocks; + + fp = fopen(MAP_FILE_NAME, "r"); + if (fp == NULL) { + LOG_ERROR("unable to get upper.map, ignored"); + return -1; + } + + while (fscanf(fp, "%" PRIx64" %x %" PRIx64 "\n", &blkaddr, &nblocks, &toff) + >= 3) + { + LSMT::RemoteMapping lba; + lba.offset = blkaddr * blksz; + lba.count = nblocks * blksz; + lba.roffset = toff; + int nwrite = fout->ioctl(LSMT::IFileRW::RemoteData, lba); + if ((unsigned) nwrite != lba.count) { + LOG_ERRNO_RETURN(0, -1, "failed to write lba"); + } + } + fclose(fp); + + return unlink(MAP_FILE_NAME); +} + +static int erofs_close_sbi(struct erofs_sb_info *sbi, ErofsCache *cache) +{ + if (cache->flush()) { + LOG_ERROR("Fail to flush caches."); + return -1; + } + return 0; +} + +static void erofs_close_tar(struct erofs_tarfile *erofstar) +{ + free(erofstar->ios.buffer); +} + +int LibErofs::extract_tar(photon::fs::IFile *source, bool meta_only, bool first_layer) +{ + struct erofs_sb_info sbi = {}; + struct erofs_tarfile erofstar = {}; + struct erofs_mkfs_cfg cfg; + int err; + + _target = target; + _source = source; + + target_vfops.pread = erofs_target_pread; + target_vfops.pwrite = erofs_target_pwrite; + target_vfops.pread = erofs_target_pread; + target_vfops.pwrite = erofs_target_pwrite; + target_vfops.fsync = erofs_target_fsync; + target_vfops.fallocate = erofs_target_fallocate; + target_vfops.ftruncate = erofs_target_ftruncate; + target_vfops.read = erofs_target_read; + target_vfops.lseek = erofs_target_lseek; + + source_vfops.pread = erofs_source_pread; + source_vfops.pwrite = erofs_source_pwrite; + source_vfops.fsync = erofs_source_fsync; + source_vfops.fallocate = erofs_source_fallocate; + source_vfops.ftruncate = erofs_source_ftruncate; + source_vfops.read = erofs_source_read; + source_vfops.lseek = erofs_source_lseek; + + erofs_cache.file = _target; + erofs_cache.capacity = 128; + + /* initialization of sbi */ + err = erofs_init_sbi(&sbi, _target, &target_vfops, ilog2(blksize)); + if (err) { + erofs_close_sbi(&sbi, &erofs_cache); + LOG_ERROR("Failed to init sbi."); + return err; + } + /* initialization of erofstar */ + err = erofs_init_tar(&erofstar, _source, &source_vfops); + if (err) { + LOG_ERROR("Failed to init tarerofs"); + goto exit; + } + + cfg.sbi = &sbi; + cfg.erofstar = &erofstar; + cfg.incremental = !first_layer; + cfg.ovlfs_strip = true; + + err = erofs_mkfs(&cfg); + if (err) { + LOG_ERROR("Failed to mkfs."); + goto exit; + } + + /* write mapfile */ + err = erofs_write_map_file(_target, blksize); + if (err) { + LOG_ERROR("Failed to write mapfile."); + goto exit; + } +exit: + err = erofs_close_sbi(&sbi, &erofs_cache); + erofs_close_tar(&erofstar); + return err; +} + +LibErofs::LibErofs(photon::fs::IFile *target, uint64_t blksize) + : target(target), blksize(blksize) +{ +} + +LibErofs::~LibErofs() +{ +} diff --git a/src/overlaybd/tar/erofs/liberofs.h b/src/overlaybd/tar/erofs/liberofs.h new file mode 100644 index 00000000..c1064d65 --- /dev/null +++ b/src/overlaybd/tar/erofs/liberofs.h @@ -0,0 +1,17 @@ +#ifndef TAREROFS_INTERFACE_H +#define TAREROFS_INTERFACE_H + +#include +#include +#include + +class LibErofs{ +public: + LibErofs(photon::fs::IFile *target, uint64_t blksize); + ~LibErofs(); + int extract_tar(photon::fs::IFile *source, bool meta_only, bool first_layer); +private: + photon::fs::IFile * target= nullptr; /* output file */ + uint64_t blksize; +}; +#endif diff --git a/src/overlaybd/tar/tarerofs.cpp b/src/overlaybd/tar/tarerofs.cpp deleted file mode 100644 index b7b20fa7..00000000 --- a/src/overlaybd/tar/tarerofs.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include "tarerofs.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../lsmt/file.h" -#include "../lsmt/index.h" - -#define TAREROFS_BLOCK_SIZE 4096 -#define __stringify_1(x...) #x -#define __stringify(x...) __stringify_1(x) - -#define LSMT_ALIGNMENT 512 - -int TarErofs::extract_all() { - ssize_t read; - struct stat st; - char buf[128*1024]; - char base_path[64] = "/tmp/tarerofs_base_XXXXXX"; - char command_line[256] = "mkfs.erofs --tar=0,upper.map,1073741824 -b" __stringify(TAREROFS_BLOCK_SIZE) " --aufs"; - const char command_line2[] = " upper.erofs"; - - FILE *fp; - photon::fs::IFile *rawfs = nullptr; - int status; - uint64_t blkaddr, toff, metasize; - uint32_t nblocks; - - if (!meta_only) { - LOG_ERROR("currently EROFS supports fastoci mode only", strerror(errno)); - return -1; - } - - if (!first_layer) { - int fd = mkstemp(base_path); - if (fd < 0) { - LOG_ERROR("cannot generate a temporary file to dump overlaybd disk"); - return -1; - } - std::strcat(command_line, " --base "); - std::strcat(command_line, base_path); - - // lsmt.pread should align to 512 - if (fs_base_file->pread(&buf, LSMT_ALIGNMENT, 0) != LSMT_ALIGNMENT) { - LOG_ERROR("failed to read EROFS metadata size"); - return -1; - } - metasize = *(uint64_t *)buf; - - while (metasize) { - int count = std::min(sizeof(buf), metasize); - read = fs_base_file->read(buf, count); - if (read != count || - write(fd, buf, read) != read) { - read = -1; - break; - } - metasize -= read; - } - close(fd); - if (read < 0) { - return -1; - } - } - std::strcat(command_line, command_line2); - fp = popen(command_line, "w"); - if (fp == NULL) { - LOG_ERROR("failed to execute mkfs.erofs", strerror(errno)); - return -1; - } - - while ((read = file->read(buf, sizeof(buf))) > 0) { - if (fwrite(buf, read, 1, fp) != 1) { - read = -1; - break; - } - } - status = pclose(fp); - - if (!first_layer) - unlink(base_path); - - if (read < 0 || status) { - return -1; - } - - rawfs = photon::fs::open_localfile_adaptor("upper.erofs", O_RDONLY, 0644); - DEFER({ delete rawfs; }); - - /* write to LSMT */ - metasize = rawfs->lseek(0, SEEK_END); - rawfs->lseek(0, 0); - fout->lseek(0, 0); - while ((read = rawfs->read(buf, sizeof(buf))) > 0) { - if (metasize) { // since pwrite < 512 is unsupported. - *(uint64_t *)buf = metasize; - metasize = 0; - } - - if (fout->write(buf, read) != read) { - read = -1; - break; - } - } - - /* write mapfile */ - fp = fopen("upper.map", "r"); - if (fp == NULL) { - LOG_ERROR("unable to get upper.map, ignored"); - return -1; - } - while (fscanf(fp, "%" PRIx64" %x %" PRIx64 "\n", &blkaddr, &nblocks, &toff) >= 3) { - LSMT::RemoteMapping lba; - lba.offset = blkaddr * TAREROFS_BLOCK_SIZE; - lba.count = nblocks * TAREROFS_BLOCK_SIZE; - lba.roffset = toff; - int nwrite = fout->ioctl(LSMT::IFileRW::RemoteData, lba); - if ((unsigned) nwrite != lba.count) { - LOG_ERRNO_RETURN(0, -1, "failed to write lba"); - } - } - fclose(fp); - return 0; -} diff --git a/src/overlaybd/tar/tarerofs.h b/src/overlaybd/tar/tarerofs.h deleted file mode 100644 index 51d61232..00000000 --- a/src/overlaybd/tar/tarerofs.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -class TarErofs { -public: - TarErofs(photon::fs::IFile *file, photon::fs::IFile *target, uint64_t fs_blocksize = 4096, - photon::fs::IFile *bf = nullptr, bool meta_only = true, bool first_layer = true) - : file(file), fout(target), fs_base_file(bf), meta_only(meta_only), first_layer(first_layer) {} - - int extract_all(); - -private: - photon::fs::IFile *file = nullptr; // source - photon::fs::IFile *fout = nullptr; // target - photon::fs::IFile *fs_base_file = nullptr; - bool meta_only; - bool first_layer; - std::set unpackedPaths; - std::list> dirs; // -}; diff --git a/src/tools/turboOCI-apply.cpp b/src/tools/turboOCI-apply.cpp index 49f410fc..340f3939 100644 --- a/src/tools/turboOCI-apply.cpp +++ b/src/tools/turboOCI-apply.cpp @@ -23,7 +23,7 @@ #include "../overlaybd/lsmt/file.h" #include "../overlaybd/zfile/zfile.h" #include "../overlaybd/tar/libtar.h" -#include "../overlaybd/tar/tarerofs.h" +#include "../overlaybd/tar/erofs/liberofs.h" #include "../overlaybd/gzindex/gzfile.h" #include "../overlaybd/gzip/gz.h" #include @@ -135,10 +135,8 @@ int main(int argc, char **argv) { exit(-1); } - auto tar = - new TarErofs(src_file, imgfile, 4096, base_file, true, cfg.lowers().size() == 0); - - if (tar->extract_all() < 0) { + auto tar = new LibErofs(imgfile, 4096); + if (tar->extract_tar(src_file, true, cfg.lowers().size() == 0) < 0) { fprintf(stderr, "failed to extract\n"); exit(-1); }