From 62e5e66051e3ac799efdeefe3486202ca3fe90c7 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Fri, 18 Oct 2024 15:59:53 +0800 Subject: [PATCH 1/2] Add EROFS test framework Add a unit test to test the recent regression. Signed-off-by: Gao Xiang --- src/overlaybd/tar/erofs/CMakeLists.txt | 4 + src/overlaybd/tar/erofs/test/CMakeLists.txt | 15 ++ src/overlaybd/tar/erofs/test/test.cpp | 263 ++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 src/overlaybd/tar/erofs/test/CMakeLists.txt create mode 100644 src/overlaybd/tar/erofs/test/test.cpp diff --git a/src/overlaybd/tar/erofs/CMakeLists.txt b/src/overlaybd/tar/erofs/CMakeLists.txt index 1b423e5c..6b1b16ee 100644 --- a/src/overlaybd/tar/erofs/CMakeLists.txt +++ b/src/overlaybd/tar/erofs/CMakeLists.txt @@ -39,3 +39,7 @@ target_include_directories(erofs_lib PRIVATE target_compile_options(erofs_lib PRIVATE "-include${EROFS_CONFIG_FILE}") target_link_libraries(erofs_lib PRIVATE ${EROFS_LIB_STATIC}) + +if(BUILD_TESTING) + add_subdirectory(test) +endif() diff --git a/src/overlaybd/tar/erofs/test/CMakeLists.txt b/src/overlaybd/tar/erofs/test/CMakeLists.txt new file mode 100644 index 00000000..290504fa --- /dev/null +++ b/src/overlaybd/tar/erofs/test/CMakeLists.txt @@ -0,0 +1,15 @@ +include_directories($ENV{GFLAGS}/include) +link_directories($ENV{GFLAGS}/lib) + +include_directories($ENV{GTEST}/googletest/include) +link_directories($ENV{GTEST}/lib) + +add_executable(erofs_test test.cpp) +target_include_directories(erofs_test PUBLIC ${PHOTON_INCLUDE_DIR}) +target_link_libraries(erofs_test gtest gtest_main pthread photon_static + tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib) + +add_test( + NAME erofs_test + COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_test +) diff --git a/src/overlaybd/tar/erofs/test/test.cpp b/src/overlaybd/tar/erofs/test/test.cpp new file mode 100644 index 00000000..f9cbde37 --- /dev/null +++ b/src/overlaybd/tar/erofs/test/test.cpp @@ -0,0 +1,263 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../gzindex/gzfile.h" +#include "../../../lsmt/file.h" +#include "../liberofs.h" +#include "../../tar_file.cpp" +#include "../../../gzip/gz.h" +#include "../../../../tools/sha256file.h" + + +#define FILE_SIZE (2 * 1024 * 1024) +#define IMAGE_SIZE 512UL<<20 +class ErofsTest : public ::testing::Test { +protected: + virtual void SetUp() override{ + fs = photon::fs::new_localfs_adaptor(); + + ASSERT_NE(nullptr, fs); + if (fs->access(workdir.c_str(), 0) != 0) { + auto ret = fs->mkdir(workdir.c_str(), 0755); + ASSERT_EQ(0, ret); + } + + fs = photon::fs::new_subfs(fs, workdir.c_str(), true); + ASSERT_NE(nullptr, fs); + } + virtual void TearDown() override{ + for (auto fn : filelist){ + fs->unlink(fn.c_str()); + } + if (fs) + delete fs; + } + + int download(const std::string &url, std::string out = "") { + if (out == "") { + out = workdir + "/" + std::string(basename(url.c_str())); + } + if (fs->access(out.c_str(), 0) == 0) + return 0; + // download file + std::string cmd = "curl -s -o " + out + " " + url; + LOG_INFO(VALUE(cmd.c_str())); + auto ret = system(cmd.c_str()); + if (ret != 0) { + LOG_ERRNO_RETURN(0, -1, "download failed: `", url.c_str()); + } + return 0; + } + + int download_decomp(const std::string &url) { + // download file + std::string cmd = "wget -q -O - " + url +" | gzip -d -c >" + + workdir + "/test.tar"; + LOG_INFO(VALUE(cmd.c_str())); + auto ret = system(cmd.c_str()); + if (ret != 0) { + LOG_ERRNO_RETURN(0, -1, "download failed: `", url.c_str()); + } + return 0; + } + + int inflate(unsigned char *data, unsigned int size) { + unsigned char out[65536]; + z_stream strm; + int ret; + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, 31); + if (ret != Z_OK) + return ret; + DEFER((void)inflateEnd(&strm)); + strm.avail_in = size; + strm.next_in = data; + int fd = open(std::string(workdir + "/test.tar").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); + if (fd < 0) + return fd; + DEFER(close(fd)); + do { + strm.avail_out = sizeof(out); + strm.next_out = out; + ret = ::inflate(&strm, Z_NO_FLUSH); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + return -1; + } + int have = sizeof(out) - strm.avail_out; + if (write(fd, out, have) != have) { + return -1; + } + } while (strm.avail_out == 0); + return 0; + } + int write_file(photon::fs::IFile *file) { + std::string bb = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01"; + ssize_t size = 0; + ssize_t ret; + struct stat st; + LOG_INFO(VALUE(bb.size())); + while (size < FILE_SIZE) { + ret = file->write(bb.data(), bb.size()); + EXPECT_EQ(bb.size(), ret); + ret = file->fstat(&st); + EXPECT_EQ(0, ret); + ret = file->lseek(0, SEEK_CUR); + EXPECT_EQ(st.st_size, ret); + size += bb.size(); + } + LOG_INFO("write ` byte", size); + EXPECT_EQ(FILE_SIZE, size); + return 0; + } + + IFile *createDevice(const char *fn, IFile *target_file, size_t virtual_size = IMAGE_SIZE){ + auto fn_idx = std::string(fn)+".idx"; + auto fn_meta = std::string(fn)+".meta"; + DEFER({ + filelist.push_back(fn_idx); + filelist.push_back(fn_meta); + }); + auto fmeta = fs->open(fn_idx.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); + auto findex = fs->open(fn_meta.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU); + LSMT::WarpFileArgs args(findex, fmeta, target_file); + args.virtual_size = virtual_size; + return create_warpfile(args, false); + } + + int do_verify(IFile *verify, IFile *test, off_t offset = 0, ssize_t count = -1) { + + if (count == -1) { + count = verify->lseek(0, SEEK_END); + auto len = test->lseek(0, SEEK_END); + if (count != len) { + LOG_ERROR("check logical length failed"); + return -1; + } + } + LOG_INFO("start verify, virtual size: `", count); + + ssize_t LEN = 1UL<<20; + char vbuf[1UL<<20], tbuf[1UL<<20]; + // set_log_output_level(0); + for (off_t i = 0; i < count; i+=LEN) { + LOG_DEBUG("`", i); + auto ret_v = verify->pread(vbuf, LEN, i); + auto ret_t = test->pread(tbuf, LEN, i); + if (ret_v == -1 || ret_t == -1) { + LOG_ERROR_RETURN(0, -1, "pread(`,`) failed. (ret_v: `, ret_t: `)", + i, LEN, ret_v, ret_v); + } + if (ret_v != ret_t) { + LOG_ERROR_RETURN(0, -1, "compare pread(`,`) return code failed. ret:` / `(expected)", + i, LEN, ret_t, ret_v); + } + if (memcmp(vbuf, tbuf, ret_v)!= 0){ + LOG_ERROR_RETURN(0, -1, "compare pread(`,`) buffer failed.", i, LEN); + } + } + return 0; + } + + std::string workdir = "/tmp/tar_test"; + photon::fs::IFileSystem *fs; + std::vector filelist; +}; + +TEST_F(ErofsTest, tar_meta) { + unsigned char tar_zipped[249] = { + 0x1f, 0x8b, 0x08, 0x08, 0x7d, 0x06, 0x12, 0x67, 0x00, 0x03, 0x74, + 0x65, 0x73, 0x74, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd7, 0x31, + 0x0e, 0xc2, 0x30, 0x0c, 0x85, 0xe1, 0xce, 0x9c, 0xa2, 0x47, 0x48, + 0x52, 0x27, 0x86, 0xe3, 0x14, 0xc4, 0x05, 0x68, 0xb8, 0x3f, 0x35, + 0xe9, 0xc0, 0xea, 0xc1, 0x2e, 0x92, 0xdf, 0xbf, 0x54, 0xea, 0xf2, + 0xb2, 0x7c, 0x52, 0xd2, 0x9f, 0x5b, 0x9f, 0x8c, 0x4b, 0x7b, 0x8d, + 0x48, 0xbe, 0x99, 0x6b, 0xfa, 0xfd, 0x1e, 0xd1, 0x94, 0x89, 0x13, + 0x51, 0xca, 0x39, 0xcb, 0xff, 0xd6, 0x5a, 0x99, 0xe6, 0x64, 0x7d, + 0x30, 0xe9, 0xbd, 0xf5, 0xf5, 0x35, 0xcf, 0x1e, 0x53, 0xff, 0xd8, + 0x7a, 0x7f, 0x5c, 0xce, 0x3e, 0x03, 0x3a, 0xaf, 0xbe, 0xfb, 0x2f, + 0xc6, 0x1b, 0x0a, 0xff, 0x85, 0x4b, 0x16, 0xff, 0x5c, 0x2b, 0xfc, + 0x7b, 0x04, 0xff, 0xb1, 0x13, 0xff, 0x8b, 0xf1, 0x86, 0xce, 0x3f, + 0x7f, 0xfd, 0x37, 0x82, 0x7f, 0x8f, 0xe0, 0x3f, 0x76, 0xe2, 0x9f, + 0x8c, 0x37, 0x54, 0xfe, 0x17, 0x1a, 0xfe, 0x17, 0xf8, 0xf7, 0x08, + 0xfe, 0x63, 0x27, 0xfe, 0xab, 0xf1, 0x86, 0xca, 0x3f, 0x8f, 0xfb, + 0x7f, 0xc3, 0xfd, 0xdf, 0x25, 0xf8, 0x8f, 0x9d, 0xf8, 0x6f, 0xc6, + 0x1b, 0x0a, 0xff, 0x4b, 0x2a, 0x65, 0xbc, 0xff, 0x71, 0xff, 0x77, + 0x09, 0xfe, 0x63, 0x27, 0xfe, 0xd9, 0x78, 0x43, 0xe7, 0x7f, 0xdc, + 0xff, 0x2b, 0xc3, 0xbf, 0x47, 0xf0, 0x1f, 0x3b, 0xf1, 0x7f, 0x35, + 0xde, 0x50, 0xf9, 0xdf, 0x1f, 0xfe, 0x78, 0xff, 0xfb, 0x05, 0xff, + 0xb1, 0x13, 0xff, 0x37, 0xe3, 0x0d, 0x8d, 0xff, 0x9c, 0x0f, 0xff, + 0x09, 0xfe, 0x3d, 0x82, 0x7f, 0x84, 0x10, 0x8a, 0xd9, 0x07, 0xbf, + 0x49, 0x1c, 0x0f, 0x00, 0x28, 0x00, 0x00 + }; + // set_log_output_level(0); + ASSERT_EQ(0, inflate(tar_zipped, sizeof(tar_zipped))); + + auto src_file = fs->open("test.tar", O_RDONLY, 0666); + ASSERT_NE(nullptr, src_file); + DEFER(delete src_file); + auto verify_dev = createDevice("verify", src_file); + auto tar = new LibErofs(verify_dev, 4096, false); + ASSERT_EQ(0, tar->extract_tar(src_file, true, true)); + delete tar; + + src_file->lseek(0, 0); + + auto tar_idx = fs->open("test.tar.meta", O_TRUNC | O_CREAT | O_RDWR, 0644); + auto imgfile = createDevice("mock", src_file); + DEFER(delete imgfile;); + auto tar2 = new UnTar(src_file, nullptr, 0, 4096, nullptr, true); + auto obj_count = tar2->dump_tar_headers(tar_idx); + EXPECT_NE(-1, obj_count); + LOG_INFO("objects count: `", obj_count); + tar_idx->lseek(0,0); + delete tar2; + + auto tar3 = new LibErofs(imgfile, 4096, true); + ASSERT_EQ(0, tar3->extract_tar(tar_idx, true, true)); + delete tar3; + EXPECT_EQ(0, do_verify(verify_dev, imgfile)); + delete tar_idx; +} + +int main(int argc, char **argv) { + + ::testing::InitGoogleTest(&argc, argv); + photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); + set_log_output_level(1); + + auto ret = RUN_ALL_TESTS(); + (void)ret; + + return 0; +} From dc073b2afdb2c4bd8089cfb87e2066ffa65fb70e Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Fri, 18 Oct 2024 12:47:03 +0800 Subject: [PATCH 2/2] [bugfix] workaround if some SegmentMapping refers to end of tar Currently it could happen on EROFS meta indexes. Signed-off-by: Gao Xiang --- src/overlaybd/lsmt/file.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/overlaybd/lsmt/file.cpp b/src/overlaybd/lsmt/file.cpp index d8336b3a..53bbfb4a 100644 --- a/src/overlaybd/lsmt/file.cpp +++ b/src/overlaybd/lsmt/file.cpp @@ -573,9 +573,19 @@ class LSMTReadOnlyFile : public IFileRW { // LOG_DEBUG("offset: `, length: `", m.moffset, size); ssize_t ret = m_files[m.tag]->pread(buf, size, m.moffset * ALIGNMENT); if (ret < size) { - LOG_ERRNO_RETURN(0, (int)ret, - "failed to read from `-th file ( ` pread return: ` < size: `)", - m.tag, m_files[m.tag], ret, size); + if (ret < 0) { + LOG_ERRNO_RETURN(0, -1, + "failed to read from `-th file ( ` pread return: ` < size: `)", + m.tag, m_files[m.tag], ret, size); + } + size_t ret2 = m_files[m.tag]->pread((char *)buf + ret, size - ret, m.moffset * ALIGNMENT + ret); + if (ret2) { + LOG_ERRNO_RETURN(0, (int)ret, + "failed to read from `-th file ( ` pread return: ` < size: `)", + m.tag, m_files[m.tag], ret, size); + } else { + memset((char *)buf + ret, 0, size - ret); + } } lsmt_io_size += ret; lsmt_io_cnt++;