Skip to content

Commit

Permalink
Last-ish round of fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@NWilson
NWilson committed Jan 17, 2025
1 parent ec9fdc8 commit 8613839
Show file tree
Hide file tree
Showing 30 changed files with 569 additions and 392 deletions.
22 changes: 0 additions & 22 deletions .github/workflows/dev.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,28 +439,6 @@ jobs:
- name: Test
run: bazelisk test //... --enable_runfiles --incompatible_strict_action_env --test_output=all

# XXX Maybe this should just go in the ManyConfig tests, actually??
# XXX JIT IS TURNED OFF - apparently it needs some fixes (not surprising)
ebcdic:
# Tests the full support for EBCDIC on a non-EBCDIC platform, using a
# hardcoded EBCDIC-1047 codepage.
name: EBCDIC
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: true

- name: Configure
run: cmake -DPCRE2_SUPPORT_JIT=OFF -DPCRE2_SUPPORT_UNICODE=OFF -DPCRE2_EBCDIC=ON -DPCRE2_EBCDIC_IGNORING_COMPILER=ON -DPCRE2_DEBUG=ON -DCMAKE_C_FLAGS="$CFLAGS_GCC_STYLE" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_BUILD_TYPE=Release -B build

- name: Build
run: cd build && make -j3

- name: Test
run: cd build && ../RunTest

heron:
# Job to verify that the tasks performed by PrepareRelease have been done. It is
# the committer's responsibility (currently) to run PrepareRelease themselves when
Expand Down
2 changes: 2 additions & 0 deletions Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,7 @@ EXTRA_DIST += \
testdata/testinput26 \
testdata/testinput27 \
testdata/testinput28 \
testdata/testinput29 \
testdata/testinputheap \
testdata/testoutput1 \
testdata/testoutput2 \
Expand Down Expand Up @@ -824,6 +825,7 @@ EXTRA_DIST += \
testdata/testoutput26 \
testdata/testoutput27 \
testdata/testoutput28 \
testdata/testoutput29 \
testdata/testoutputheap-16 \
testdata/testoutputheap-32 \
testdata/testoutputheap-8 \
Expand Down
117 changes: 45 additions & 72 deletions RunTest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ title25="Test 25: UTF pattern conversion tests"
title26="Test 26: Unicode property tests (compatible with Perl >= 5.38)"
title27="Test 27: Auto-generated unicode property tests"
title28="Test 28: EBCDIC-specific tests"
maxtest=28
title29="Test 29: EBCDIC-specific tests (for NL=0x25)"
maxtest=29
titleheap="Test 'heap': Environment-specific heap tests"

if [ $# -eq 1 -a "$1" = "list" ]; then
Expand Down Expand Up @@ -118,6 +119,7 @@ if [ $# -eq 1 -a "$1" = "list" ]; then
echo $title26
echo $title27
echo $title28
echo $title29
echo ""
echo $titleheap
echo ""
Expand Down Expand Up @@ -169,7 +171,10 @@ checkresult()
esac
cf_out="$testdata/testoutput$2"
if [ $ebcdic -eq 1 ] ; then
sed '/^# if !EBCDIC/,/^# endif/d' "$cf_out" >testtry2
# We currently only use the #if ... #endif support in pcre2test for EBCDIC
# testing. Run in "preprocess-only" mode (-E) on the testoutput file to trim
# the output lines matching the input lines which are discarded.
$sim $pcre2test -q -E "$cf_out" >testtry2
cf_out=testtry2
fi
$cf "$cf_out" testtry
Expand Down Expand Up @@ -249,6 +254,7 @@ do25=no
do26=no
do27=no
do28=no
do29=no
doheap=no

while [ $# -gt 0 ] ; do
Expand Down Expand Up @@ -282,6 +288,7 @@ while [ $# -gt 0 ] ; do
26) do26=yes;;
27) do27=yes;;
28) do28=yes;;
29) do29=yes;;
heap) doheap=yes;;
-8) arg8=yes;;
-16) arg16=yes;;
Expand Down Expand Up @@ -358,53 +365,25 @@ support32=$?
$sim $pcre2test -C backslash-C >/dev/null
supportBSC=$?

# Check if compiled in EBCDIC mode, and whether we have EBCDIC I/O
# Check if compiled in EBCDIC mode, and whether we have EBCDIC I/O and NL=0x25
$sim $pcre2test -C ebcdic >/dev/null
ebcdic=$?
$sim $pcre2test -C ebcdic-io >/dev/null
ebcdic_io=$?
$sim $pcre2test -C ebcdic-nl25 >/dev/null
ebcdic_nl25=$?

if [ $ebcdic -eq 1 ]; then
# XXX This is a bit unholy. How about I just bite the bullet and implement
# the "#if" stuff directly inside pcre2test? It wouldn't be totally crazy.
# This does seem to work fine here though.
#
# Purpose of code: sneakily parse the pcre2test commandline, to find the
# filename of the input file. Run that through sed, to strip out the chunks
# blocked out by "# if !EBCDIC...#endif", and then invoke pcre2test with the
# modified input file. That's it.
filterebcdic()
{
filter_args=
while [ "$1" != "$pcre2test" ] ; do
# Shift off any Valgrind args
filter_args="$filter_args $1"
shift
done
# Shift off the $pcre2test arg
filter_args="$filter_args $1"
shift
while [ $# -gt 0 ] ; do
case "$1" in
-C|-error|-pattern|-S|-subject|-t|-tm|-T|-TM)
if [ $# -gt 1 ] ; then
filter_args="$filter_args $1 $2"; shift; shift
else
filter_args="$filter_args $1"; shift
fi
;;
-*) filter_args="$filter_args $1"; shift;;
*) break;;
esac
done
if [ $# -gt 0 ] ; then
input="$1"
shift
sed '/^# if !EBCDIC/,/^# endif/d' "$input" | $filter_args "/dev/stdin" "$@"
else
$filter_args "$@"
fi
}
sim="filterebcdic $sim"
if [ $ebcdic_io -eq 0 ]; then
echo "Running tests in EBCDIC mode, and expecting ASCII test data"
else
echo "Running tests in EBCDIC mode, and expecting EBCDIC test data"
echo "If you are on an EBCDIC machine, you will need to convert the PCRE2"
echo "testdata/ directory from ISO8859-1, so the data match the EBCDIC"
echo "codepage that your C compiler is using for C character literals."
echo "For example:"
echo " iconv -f ISO8859-1 -t IBM-1047 ..."
fi
fi

# Initialize all bitsizes skipped
Expand Down Expand Up @@ -483,7 +462,7 @@ if [ $do0 = no -a $do1 = no -a $do2 = no -a $do3 = no -a \
$do16 = no -a $do17 = no -a $do18 = no -a $do19 = no -a \
$do20 = no -a $do21 = no -a $do22 = no -a $do23 = no -a \
$do24 = no -a $do25 = no -a $do26 = no -a $do27 = no -a \
$do28 = no -a $doheap = no \
$do28 = no -a $do29 = no -a $doheap = no \
]; then
do0=yes
do1=yes
Expand Down Expand Up @@ -514,6 +493,7 @@ if [ $do0 = no -a $do1 = no -a $do2 = no -a $do3 = no -a \
do26=yes
do27=yes
do28=yes
do29=yes
fi

# Handle any explicit skips at this stage, so that an argument list may consist
Expand Down Expand Up @@ -980,33 +960,26 @@ for bmode in "$test8" "$test16" "$test32"; do
if [ $ebcdic -eq 0 ] ; then
echo " Skipped when not targetting EBCDIC"
else
if [ $ebcdic_io -eq 0 ] ; then
# Our testdata files are in ASCII, and the pcre2test program is using
# ASCII input: all easy.
for opt in "" "-dfa"; do
$sim $valgrind $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testtry
checkresult $? 28 "$opt"
done
else
echo "Cannot run EBCDIC tests:"
echo " Ironically this particular test script does not automatically"
echo " run the tests on an actual EBCDIC system. The testdata files"
echo " shipped with PCRE2 are in ASCII."
echo " You may be able to run the tests manually if you know which"
echo " EBCDIC codepage you used when compiling PCRE2, and then convert"
echo " the testdata to match. For example, if the C compiler used to build"
echo " PCRE2 was using IBM-1047:"
echo ""
echo " iconv -f ISO8859-1 -t IBM-1047 <testdata/testinputEBC >testinputEBC-native"
echo " pcre2test -q -$bmode testinputEBC-native >testoutputEBC-native"
echo " [ $? -eq 0 ] || echo 'pcre2test failed'"
echo " iconv -f IBM-1047 -t ISO8859-1 <testoutputEBC-native >testoutputEBC-ascii"
echo " $cf testdata/testoutputEBC testoutputEBC-ascii"
echo ""
echo "This is speculative. The PCRE2 maintainers do not have access to an"
echo "EBCDIC system to test this. Please report back if you try it."
exit 1
fi
for opt in "" $jitopt "-dfa"; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput28 testtry
checkresult $? 28 "$opt"
done
fi
fi

# EBCDIC tests (for NL=0x25)

if [ $do29 = yes ] ; then
echo $title29
if [ $ebcdic -eq 0 ] ; then
echo " Skipped when not targetting EBCDIC"
elif [ $ebcdic_nl25 -eq 0 ] ; then
echo " Skipped because EBCDIC newline is not 0x25"
else
for opt in "" $jitopt "-dfa"; do
$sim $valgrind ${opt:+$vjs} $pcre2test -q $setstack $bmode $opt $testdata/testinput29 testtry
checkresult $? 29 "$opt"
done
fi
fi

Expand Down
55 changes: 49 additions & 6 deletions RunTest.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ set unicode=%ERRORLEVEL%
set jit=%ERRORLEVEL%
%pcre2test% -C backslash-C >NUL
set supportBSC=%ERRORLEVEL%
%pcre2test% -C ebcdic >NUL
set ebcdic=%ERRORLEVEL%
%pcre2test% -C ebcdic-nl25 >NUL
set ebcdic_nl25=%ERRORLEVEL%

if %support8% EQU 1 (
if not exist testout8 md testout8
Expand Down Expand Up @@ -114,19 +118,20 @@ set do24=no
set do25=no
set do26=no
set do27=no
@rem XXX ADD TEST 28
set do28=no
set do29=no
set all=yes

for %%a in (%*) do (
set valid=no
for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27) do if %%v == %%a set valid=yes
for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29) do if %%v == %%a set valid=yes
if "!valid!" == "yes" (
set do%%a=yes
set all=no
) else (
echo Invalid test number - %%a!
echo Usage %0 [ test_number ] ...
echo Where test_number is one or more optional test numbers 1 through 27, default is all tests.
echo Where test_number is one or more optional test numbers 1 through 29, default is all tests.
exit /b 1
)
)
Expand Down Expand Up @@ -160,6 +165,8 @@ if "%all%" == "yes" (
set do25=yes
set do26=yes
set do27=yes
set do28=yes
set do29=yes
)

@echo RunTest.bat's pcre2test output is written to newly created subfolders
Expand Down Expand Up @@ -215,6 +222,8 @@ if "%do24%" == "yes" call :do24
if "%do25%" == "yes" call :do25
if "%do26%" == "yes" call :do26
if "%do27%" == "yes" call :do27
if "%do28%" == "yes" call :do28
if "%do29%" == "yes" call :do29
:modeSkip
if "%mode%" == "" (
set mode=-16
Expand Down Expand Up @@ -293,18 +302,28 @@ if errorlevel 1 (
%pcre2test% %mode% %4 %5 %6 %7 %8 %9 -error -80,-62,-2,-1,0,100,101,191,300 >>%2%bits%\%testoutput%
)

fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput% >NUL
set testexpected=%srcdir%\testdata\%testoutput%

if %ebcdic% EQU 1 (
@rem We currently only use the #if ... #endif support in pcre2test for EBCDIC
@rem testing. Run in "preprocess-only" mode (-E) on the testoutput file to trim
@rem the output lines matching the input lines which are discarded.
%pcre2test% -q -E %testexpected% >%2%bits%\%testoutput%-trimmed
set testexpected=%2%bits%\%testoutput%-trimmed
)

fc /n %testexpected% %2%bits%\%testoutput% >NUL

if errorlevel 1 (
echo. failed comparison: fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput%
echo. failed comparison: fc /n %testexpected% %2%bits%\%testoutput%
if [%1]==[3] (
echo.
echo ** Test 3 failure usually means french locale is not
echo ** available on the system, rather than a bug or problem with PCRE2.
echo.
goto :eof
)
fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput%
fc /n %testexpected% %2%bits%\%testoutput%

set failed="yes"
goto :eof
Expand Down Expand Up @@ -544,6 +563,30 @@ if %unicode% EQU 0 (
if %jit% EQU 1 call :runsub 27 testoutjit "Test with JIT Override" -q -jit
goto :eof

:do28
if %ebcdic% EQU 0 (
echo Test 28 Skipped when not targetting EBCDIC.
goto :eof
)
call :runsub 28 testout "EBCDIC-specific tests" -q
call :runsub 28 testout "EBCDIC-specific tests (DFA)" -q -dfa
if %jit% EQU 1 call :runsub 28 testoutjit "Test with JIT Override" -q -jit
goto :eof

:do29
if %ebcdic% EQU 0 (
echo Test 29 Skipped when not targetting EBCDIC.
goto :eof
)
if %ebcdic_nl25% EQU 0 (
echo Test 29 Skipped because EBCDIC newline is not 0x25.
goto :eof
)
call :runsub 29 testout "EBCDIC-specific tests for NL=0x25" -q
call :runsub 29 testout "EBCDIC-specific tests for NL=0x25 (DFA)" -q -dfa
if %jit% EQU 1 call :runsub 29 testoutjit "Test with JIT Override" -q -jit
goto :eof

:conferror
@echo.
@echo Either your build is incomplete or you have a configuration error.
Expand Down
2 changes: 1 addition & 1 deletion configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1169,7 +1169,7 @@ AC_CONFIG_COMMANDS([script-chmod], [chmod a+x pcre2-config])

# Make sure that pcre2_chartables.c is removed in case the method for
# creating it was changed by reconfiguration.
AC_CONFIG_COMMANDS([delete-old-chartables], [rm -f pcre2_chartables.c])
AC_CONFIG_COMMANDS([delete-old-chartables], [rm -f src/pcre2_chartables.c])

# Handle --enable-Werror/errwarn. This must come last, so that we don't mess
# with any of the library-detection tests (or similar).
Expand Down
Loading

0 comments on commit 8613839

Please sign in to comment.