From cd99deeda69945eddd3d595bf14288de003ce784 Mon Sep 17 00:00:00 2001 From: Azeem Sajid Date: Tue, 12 Nov 2024 18:47:22 +0500 Subject: [PATCH] Debug count benchmarks for Linux builds --- .github/workflows/ci.yml | 571 ++++++++++++++++++++------------------- .gitignore | 2 + app/count.c | 7 +- scripts/ci-build.sh | 2 +- src/zsv.c | 15 +- src/zsv_internal.c | 30 +- 6 files changed, 316 insertions(+), 311 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 500bfe35..6714fbed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,66 +61,68 @@ jobs: TAG: ${{ startsWith(github.ref, 'refs/tags/v') && github.ref_name || '' }} run: ./scripts/ci-set-tag-output-parameter.sh - clang-format: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run clang-format - run: | - sudo ln -sf /usr/bin/clang-format-15 /usr/bin/clang-format - ./scripts/ci-run-clang-format.sh - - cppcheck: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install cppcheck - run: | - sudo apt update - sudo apt install -y cppcheck - cppcheck --version - - - name: Run cppcheck - run: ./scripts/ci-run-cppcheck.sh - - - name: Upload (${{ env.CPPCHECK_XML_ARTIFACT_NAME }}) - uses: actions/upload-artifact@v4 - with: - name: ${{ env.CPPCHECK_XML_ARTIFACT_NAME }} - path: ${{ env.CPPCHECK_XML_ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload (${{ env.CPPCHECK_HTML_ARTIFACT_NAME }}) - uses: actions/upload-artifact@v4 - with: - name: ${{ env.CPPCHECK_HTML_ARTIFACT_NAME }} - path: ${{ env.CPPCHECK_HTML_ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - shellcheck: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run shellcheck - run: ./scripts/ci-run-shellcheck.sh + # clang-format: + # runs-on: ubuntu-22.04 + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Run clang-format + # run: | + # sudo ln -sf /usr/bin/clang-format-15 /usr/bin/clang-format + # ./scripts/ci-run-clang-format.sh + + # cppcheck: + # runs-on: ubuntu-latest + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Install cppcheck + # run: | + # sudo apt update + # sudo apt install -y cppcheck + # cppcheck --version + + # - name: Run cppcheck + # run: ./scripts/ci-run-cppcheck.sh + + # - name: Upload (${{ env.CPPCHECK_XML_ARTIFACT_NAME }}) + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.CPPCHECK_XML_ARTIFACT_NAME }} + # path: ${{ env.CPPCHECK_XML_ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload (${{ env.CPPCHECK_HTML_ARTIFACT_NAME }}) + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.CPPCHECK_HTML_ARTIFACT_NAME }} + # path: ${{ env.CPPCHECK_HTML_ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # shellcheck: + # runs-on: ubuntu-latest + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Run shellcheck + # run: ./scripts/ci-run-shellcheck.sh ci: - needs: [tag, clang-format, cppcheck, shellcheck] + # needs: [tag, clang-format, cppcheck, shellcheck] + needs: [tag] strategy: matrix: - os: [ubuntu-20.04, macos-13, macos-14] + # os: [ubuntu-20.04, macos-13, macos-14] + os: [ubuntu-20.04] runs-on: ${{ matrix.os }} @@ -347,175 +349,176 @@ jobs: TRIPLET: ${{ env.AMD64_MACOSX_GCC }} run: ./scripts/ci-update-homebrew-tap.sh - ci-bsd: - needs: [tag, clang-format, cppcheck, shellcheck] - runs-on: ubuntu-latest - - env: - TAG: ${{ needs.tag.outputs.TAG }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Build (${{ env.AMD64_FREEBSD_GCC }}) - uses: cross-platform-actions/action@v0.25.0 - env: - PREFIX: ${{ env.AMD64_FREEBSD_GCC }} - CC: gcc - MAKE: gmake - RUN_TESTS: true - with: - operating_system: freebsd - version: '13.2' - environment_variables: 'PREFIX CC MAKE RUN_TESTS ARTIFACT_DIR' - shell: sh - run: | - ./scripts/ci-freebsd-setup.sh - ./scripts/ci-build.sh - - - name: Prepare build artifacts for upload - run: ./scripts/ci-prepare-artifacts-for-upload.sh - - - name: Attest build artifacts for release - if: startsWith(github.ref, 'refs/tags/v') - uses: actions/attest-build-provenance@v1 - with: - subject-path: ${{ env.ARTIFACT_DIR }}/* - - - name: Verify attestations of release artifacts - if: startsWith(github.ref, 'refs/tags/v') - run: ./scripts/ci-verify-attestations.sh - - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.zip) - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.zip - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.tar.gz) - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.tar.gz - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload release artifacts - if: startsWith(github.ref, 'refs/tags/v') - run: ./scripts/ci-upload-release-artifacts.sh - - ci-mingw: - needs: [tag, clang-format, cppcheck, shellcheck] - runs-on: ubuntu-latest - - env: - TAG: ${{ needs.tag.outputs.TAG }} - - steps: - - name: Set up apt dependencies - run: | - sudo apt update - sudo apt install -y mingw-w64 nuget - sudo apt remove -y jq - - - name: Set VCPKG_MINGW_INSTALL_ROOT env var - run: echo "VCPKG_MINGW_INSTALL_ROOT=$VCPKG_INSTALLATION_ROOT/installed/x64-mingw-static" >>"$GITHUB_ENV" - - - name: Cache ncurses - uses: actions/cache@v4 - id: cache-ncurses - with: - key: ncurses:x64-mingw-static - path: ${{ env.VCPKG_MINGW_INSTALL_ROOT }} - - - name: Install ncurses with wide character support using vcpkg - if: ${{ steps.cache-ncurses.outputs.cache-hit != 'true' }} - run: | - NCURSES_PORTFILE="$VCPKG_INSTALLATION_ROOT/ports/ncurses/portfile.cmake" - cd "$VCPKG_INSTALLATION_ROOT" - if ! grep -- "--enable-widec" "$NCURSES_PORTFILE" >/dev/null; then - sed 's|--enable-pc-files|--enable-pc-files --enable-widec|' -i "$NCURSES_PORTFILE" - fi - ./vcpkg install ncurses:x64-mingw-static - tree ./installed/x64-mingw-static - - - name: Checkout - uses: actions/checkout@v4 - - - name: Build (${{ env.AMD64_WINDOWS_MINGW }}) - env: - PREFIX: ${{ env.AMD64_WINDOWS_MINGW }} - CC: x86_64-w64-mingw32-gcc - MAKE: make - RUN_TESTS: false - CXX: x86_64-w64-mingw32-g++ - CPP: x86_64-w64-mingw32-cpp - RANLIB: x86_64-w64-mingw32-gcc-ranlib - AR: x86_64-w64-mingw32-gcc-ar - NM: x86_64-w64-mingw32-gcc-nm - WINDRES: x86_64-w64-mingw32-windres - CFLAGS: -I${{ env.VCPKG_MINGW_INSTALL_ROOT }}/include - LDFLAGS: -L${{ env.VCPKG_MINGW_INSTALL_ROOT }}/lib - run: | - ./scripts/ci-build.sh - ./scripts/ci-create-nuget-package.sh - - - name: Prepare build artifacts for upload - run: ./scripts/ci-prepare-artifacts-for-upload.sh - - - name: Attest build artifacts for release - if: startsWith(github.ref, 'refs/tags/v') - uses: actions/attest-build-provenance@v1 - with: - subject-path: ${{ env.ARTIFACT_DIR }}/* - - - name: Verify attestations of release artifacts - if: startsWith(github.ref, 'refs/tags/v') - run: ./scripts/ci-verify-attestations.sh - - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.zip) - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.zip - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.tar.gz) - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.tar.gz - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.nupkg) - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.nupkg - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - - name: Upload release artifacts - if: startsWith(github.ref, 'refs/tags/v') - run: ./scripts/ci-upload-release-artifacts.sh + # ci-bsd: + # needs: [tag, clang-format, cppcheck, shellcheck] + # runs-on: ubuntu-latest + + # env: + # TAG: ${{ needs.tag.outputs.TAG }} + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Build (${{ env.AMD64_FREEBSD_GCC }}) + # uses: cross-platform-actions/action@v0.25.0 + # env: + # PREFIX: ${{ env.AMD64_FREEBSD_GCC }} + # CC: gcc + # MAKE: gmake + # RUN_TESTS: true + # with: + # operating_system: freebsd + # version: '13.2' + # environment_variables: 'PREFIX CC MAKE RUN_TESTS ARTIFACT_DIR' + # shell: sh + # run: | + # ./scripts/ci-freebsd-setup.sh + # ./scripts/ci-build.sh + + # - name: Prepare build artifacts for upload + # run: ./scripts/ci-prepare-artifacts-for-upload.sh + + # - name: Attest build artifacts for release + # if: startsWith(github.ref, 'refs/tags/v') + # uses: actions/attest-build-provenance@v1 + # with: + # subject-path: ${{ env.ARTIFACT_DIR }}/* + + # - name: Verify attestations of release artifacts + # if: startsWith(github.ref, 'refs/tags/v') + # run: ./scripts/ci-verify-attestations.sh + + # - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.zip) + # uses: actions/upload-artifact@v4 + # env: + # ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.zip + # with: + # name: ${{ env.ARTIFACT_NAME }} + # path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.tar.gz) + # uses: actions/upload-artifact@v4 + # env: + # ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_FREEBSD_GCC }}.tar.gz + # with: + # name: ${{ env.ARTIFACT_NAME }} + # path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload release artifacts + # if: startsWith(github.ref, 'refs/tags/v') + # run: ./scripts/ci-upload-release-artifacts.sh + + # ci-mingw: + # needs: [tag, clang-format, cppcheck, shellcheck] + # runs-on: ubuntu-latest + + # env: + # TAG: ${{ needs.tag.outputs.TAG }} + + # steps: + # - name: Set up apt dependencies + # run: | + # sudo apt update + # sudo apt install -y mingw-w64 nuget + # sudo apt remove -y jq + + # - name: Set VCPKG_MINGW_INSTALL_ROOT env var + # run: echo "VCPKG_MINGW_INSTALL_ROOT=$VCPKG_INSTALLATION_ROOT/installed/x64-mingw-static" >>"$GITHUB_ENV" + + # - name: Cache ncurses + # uses: actions/cache@v4 + # id: cache-ncurses + # with: + # key: ncurses:x64-mingw-static + # path: ${{ env.VCPKG_MINGW_INSTALL_ROOT }} + + # - name: Install ncurses with wide character support using vcpkg + # if: ${{ steps.cache-ncurses.outputs.cache-hit != 'true' }} + # run: | + # NCURSES_PORTFILE="$VCPKG_INSTALLATION_ROOT/ports/ncurses/portfile.cmake" + # cd "$VCPKG_INSTALLATION_ROOT" + # if ! grep -- "--enable-widec" "$NCURSES_PORTFILE" >/dev/null; then + # sed 's|--enable-pc-files|--enable-pc-files --enable-widec|' -i "$NCURSES_PORTFILE" + # fi + # ./vcpkg install ncurses:x64-mingw-static + # tree ./installed/x64-mingw-static + + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Build (${{ env.AMD64_WINDOWS_MINGW }}) + # env: + # PREFIX: ${{ env.AMD64_WINDOWS_MINGW }} + # CC: x86_64-w64-mingw32-gcc + # MAKE: make + # RUN_TESTS: false + # CXX: x86_64-w64-mingw32-g++ + # CPP: x86_64-w64-mingw32-cpp + # RANLIB: x86_64-w64-mingw32-gcc-ranlib + # AR: x86_64-w64-mingw32-gcc-ar + # NM: x86_64-w64-mingw32-gcc-nm + # WINDRES: x86_64-w64-mingw32-windres + # CFLAGS: -I${{ env.VCPKG_MINGW_INSTALL_ROOT }}/include + # LDFLAGS: -L${{ env.VCPKG_MINGW_INSTALL_ROOT }}/lib + # run: | + # ./scripts/ci-build.sh + # ./scripts/ci-create-nuget-package.sh + + # - name: Prepare build artifacts for upload + # run: ./scripts/ci-prepare-artifacts-for-upload.sh + + # - name: Attest build artifacts for release + # if: startsWith(github.ref, 'refs/tags/v') + # uses: actions/attest-build-provenance@v1 + # with: + # subject-path: ${{ env.ARTIFACT_DIR }}/* + + # - name: Verify attestations of release artifacts + # if: startsWith(github.ref, 'refs/tags/v') + # run: ./scripts/ci-verify-attestations.sh + + # - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.zip) + # uses: actions/upload-artifact@v4 + # env: + # ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.zip + # with: + # name: ${{ env.ARTIFACT_NAME }} + # path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.tar.gz) + # uses: actions/upload-artifact@v4 + # env: + # ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.tar.gz + # with: + # name: ${{ env.ARTIFACT_NAME }} + # path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.nupkg) + # uses: actions/upload-artifact@v4 + # env: + # ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_WINDOWS_MINGW }}.nupkg + # with: + # name: ${{ env.ARTIFACT_NAME }} + # path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + # retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + # if-no-files-found: error + + # - name: Upload release artifacts + # if: startsWith(github.ref, 'refs/tags/v') + # run: ./scripts/ci-upload-release-artifacts.sh ci-musl: - needs: [tag, clang-format, cppcheck, shellcheck] + # needs: [tag, clang-format, cppcheck, shellcheck] + needs: [tag] runs-on: ubuntu-latest container: alpine:latest @@ -576,65 +579,65 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') run: ./scripts/ci-upload-release-artifacts.sh - ghcr: - needs: [tag, ci-musl] - runs-on: ubuntu-latest - - permissions: - packages: write - - env: - TAG: ${{ needs.tag.outputs.TAG }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - sparse-checkout: | - Dockerfile.ci - - - name: Download (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip) - uses: actions/download-artifact@v4 - with: - name: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip - path: ${{ env.AMD64_LINUX_MUSL }} - - - name: Unzip - env: - ZIP: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip - DIR: ${{ env.AMD64_LINUX_MUSL }} - run: | - cd "$DIR" - unzip -o "$ZIP" - cd .. - mkdir -p ./ci - mv ./"$DIR"/bin/zsv ./ci/ - rm -rf ./"$DIR" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push (on release) - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_RECORD_UPLOAD: false - with: - no-cache: true - context: . - file: Dockerfile.ci - platforms: linux/amd64 - push: ${{ startsWith(github.ref, 'refs/tags/v') }} - tags: | - ghcr.io/liquidaty/zsv:${{ env.TAG }} - ghcr.io/liquidaty/zsv:latest + # ghcr: + # needs: [tag, ci-musl] + # runs-on: ubuntu-latest + + # permissions: + # packages: write + + # env: + # TAG: ${{ needs.tag.outputs.TAG }} + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # sparse-checkout: | + # Dockerfile.ci + + # - name: Download (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip) + # uses: actions/download-artifact@v4 + # with: + # name: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip + # path: ${{ env.AMD64_LINUX_MUSL }} + + # - name: Unzip + # env: + # ZIP: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip + # DIR: ${{ env.AMD64_LINUX_MUSL }} + # run: | + # cd "$DIR" + # unzip -o "$ZIP" + # cd .. + # mkdir -p ./ci + # mv ./"$DIR"/bin/zsv ./ci/ + # rm -rf ./"$DIR" + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Login to GitHub Container Registry + # if: ${{ startsWith(github.ref, 'refs/tags/v') }} + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Build and push (on release) + # uses: docker/build-push-action@v6 + # env: + # DOCKER_BUILD_RECORD_UPLOAD: false + # with: + # no-cache: true + # context: . + # file: Dockerfile.ci + # platforms: linux/amd64 + # push: ${{ startsWith(github.ref, 'refs/tags/v') }} + # tags: | + # ghcr.io/liquidaty/zsv:${{ env.TAG }} + # ghcr.io/liquidaty/zsv:latest diff --git a/.gitignore b/.gitignore index 916d5d8e..6728423f 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,5 @@ data/loans_2.csv data/.zsv/data/loans_2.csv/overwrite.sqlite3 compile_commands.json .cache +zsvsheet_filter_* +overwrite.sqlite3 diff --git a/app/count.c b/app/count.c index 6626d1ea..2114fe9d 100644 --- a/app/count.c +++ b/app/count.c @@ -40,7 +40,6 @@ static int count_usage() { int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts, struct zsv_prop_handler *custom_prop_handler, const char *opts_used) { - struct data data = {0}; const char *input_path = NULL; int err = 0; for (int i = 1; !err && i < argc; i++) { @@ -77,14 +76,14 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op #endif if (!err) { - opts->row_handler = opts->verbose ? row_verbose : row; + struct data data = {0}; opts->ctx = &data; + opts->row_handler = opts->verbose ? row_verbose : row; if (zsv_new_with_properties(opts, custom_prop_handler, input_path, opts_used, &data.parser) != zsv_status_ok) { fprintf(stderr, "Unable to initialize parser\n"); err = 1; } else { - enum zsv_status status; - while ((status = zsv_parse_more(data.parser)) == zsv_status_ok) + while (zsv_parse_more(data.parser) == zsv_status_ok) ; zsv_finish(data.parser); zsv_delete(data.parser); diff --git a/scripts/ci-build.sh b/scripts/ci-build.sh index d5f39851..7114084d 100755 --- a/scripts/ci-build.sh +++ b/scripts/ci-build.sh @@ -79,7 +79,7 @@ fi if [ "$RUN_TESTS" = true ]; then echo "[INF] Running tests" rm -rf build "$PREFIX" - "$MAKE" test + "$MAKE" -C app/test test-count echo "[INF] Tests completed successfully!" if [ "$(echo "$LDFLAGS" | grep -- "-static")" != "" ] || [ "$STATIC_BUILD" = "1" ]; then diff --git a/src/zsv.c b/src/zsv.c index fc1f4fc5..7ff47d6a 100644 --- a/src/zsv.c +++ b/src/zsv.c @@ -107,7 +107,7 @@ enum zsv_status zsv_parse_more(struct zsv_scanner *scanner) { if (VERY_UNLIKELY(scanner->insert_string != NULL)) zsv_insert_string(scanner); - size_t capacity = scanner_pre_parse(scanner); + const size_t capacity = scanner_pre_parse(scanner); size_t bytes_read; if (VERY_UNLIKELY(scanner->checked_bom == 0)) { #ifdef ZSV_EXTRAS @@ -115,17 +115,16 @@ enum zsv_status zsv_parse_more(struct zsv_scanner *scanner) { if (scanner->opts.progress.seconds_interval) scanner->progress.last_time = time(NULL); #endif - size_t bom_len = strlen(ZSV_BOM); scanner->checked_bom = 1; - if ((bytes_read = scanner->read(scanner->buff.buff, 1, bom_len, scanner->in)) == bom_len && - !memcmp(scanner->buff.buff, ZSV_BOM, bom_len)) { + if ((bytes_read = scanner->read(scanner->buff.buff, 1, ZSV_BOM_LEN, scanner->in)) == ZSV_BOM_LEN && + !memcmp(scanner->buff.buff, ZSV_BOM, ZSV_BOM_LEN)) { // have bom. disregard what we just read bytes_read = scanner->read(scanner->buff.buff, 1, capacity, scanner->in); scanner->had_bom = 1; } else { // no BOM. keep the bytes we just read - // bytes_read = bom_len + scanner->read(scanner->buff.buff + bom_len, 1, capacity - bom_len, scanner->in); - if (bytes_read == bom_len) // maybe we only read < 3 bytes - bytes_read += scanner->read(scanner->buff.buff + bom_len, 1, capacity - bom_len, scanner->in); + // bytes_read = ZSV_BOM_LEN + scanner->read(scanner->buff.buff + ZSV_BOM_LEN, 1, capacity - ZSV_BOM_LEN, scanner->in); + if (bytes_read == ZSV_BOM_LEN) // maybe we only read < 3 bytes + bytes_read += scanner->read(scanner->buff.buff + ZSV_BOM_LEN, 1, capacity - ZSV_BOM_LEN, scanner->in); } } else // already checked bom. read as usual bytes_read = scanner->read(scanner->buff.buff + scanner->partial_row_length, 1, capacity, scanner->in); @@ -446,7 +445,7 @@ size_t zsv_scanned_length(zsv_parser parser) { ZSV_EXPORT size_t zsv_cum_scanned_length(zsv_parser parser) { return parser->cum_scanned_length + (parser->finished ? 0 : parser->scanned_length) + - (parser->had_bom ? strlen(ZSV_BOM) : 0); + (parser->had_bom ? ZSV_BOM_LEN : 0); } ZSV_EXPORT diff --git a/src/zsv_internal.c b/src/zsv_internal.c index 3fa392a1..05a5db55 100644 --- a/src/zsv_internal.c +++ b/src/zsv_internal.c @@ -290,7 +290,7 @@ __attribute__((always_inline)) static inline void cell_dl(struct zsv_scanner *sc } else { if (scanner->quote_close_position) { // the first char was a quote, and we have content after the closing quote - // the solution below is a generalized on that will work + // the solution below is a generalized one that will work // for the easy and usual case, but by handling separately // we avoid the memmove in the easy / usual case memmove(s + 1, s, scanner->quote_close_position); @@ -308,24 +308,25 @@ __attribute__((always_inline)) static inline void cell_dl(struct zsv_scanner *sc } } } - } else if (UNLIKELY(scanner->opts.delimiter != ',')) { - if (memchr(s, ',', n)) - scanner->quoted = ZSV_PARSER_QUOTE_NEEDED; } + // } else if (UNLIKELY(scanner->opts.delimiter != ',')) { + // if (memchr(s, ',', n)) + // scanner->quoted = ZSV_PARSER_QUOTE_NEEDED; + // } // end quote handling - if (scanner->opts.malformed_utf8_replace) { - if (scanner->opts.malformed_utf8_replace < 0) - n = zsv_strencode(s, n, 0, NULL, NULL); - else - n = zsv_strencode(s, n, scanner->opts.malformed_utf8_replace, NULL, NULL); - } + // if (scanner->opts.malformed_utf8_replace) { + // if (scanner->opts.malformed_utf8_replace < 0) + // n = zsv_strencode(s, n, 0, NULL, NULL); + // else + // n = zsv_strencode(s, n, scanner->opts.malformed_utf8_replace, NULL, NULL); + // } - if (UNLIKELY(scanner->opts.cell_handler != NULL)) - scanner->opts.cell_handler(scanner->opts.ctx, s, n); + // if (UNLIKELY(scanner->opts.cell_handler != NULL)) + // scanner->opts.cell_handler(scanner->opts.ctx, s, n); if (VERY_LIKELY(scanner->row.used < scanner->row.allocated)) { struct zsv_row *row = &scanner->row; - struct zsv_cell c = {s, n, scanner->opts.no_quotes ? 1 : scanner->quoted, 0}; + const struct zsv_cell c = {s, n, scanner->opts.no_quotes ? 1 : scanner->quoted, 0}; row->cells[row->used++] = c; } else scanner->row.overflow++; @@ -476,6 +477,7 @@ static enum zsv_status zsv_scan(struct zsv_scanner *scanner, unsigned char *buff } #define ZSV_BOM "\xef\xbb\xbf" +#define ZSV_BOM_LEN (sizeof(ZSV_BOM) - 1) // optional: set a filter function to filter data before it is processed // function should return the number of bytes to process. this may be smaller @@ -648,7 +650,7 @@ static int zsv_scanner_init(struct zsv_scanner *scanner, struct zsv_opts *opts) need_buff_size = opts->max_row_size * 2; opts->delimiter = opts->delimiter ? opts->delimiter : ','; if (opts->delimiter == '\n' || opts->delimiter == '\r' || opts->delimiter == '"') { - fprintf(stderr, "warning: ignoring illegal delimiter\n"); + fprintf(stderr, "Warning: ignoring illegal delimiter\n"); opts->delimiter = ','; }