From 382d1e293a1ddf268d62e993e4f7ae3c75501633 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 10 Nov 2024 11:21:53 -0500 Subject: [PATCH 1/2] Fix CPP conditional scanner (cpp.py) and test suite (cppTests.py). Changes: - Preserve non-integer literals that contain valid integer specifications. - Add binary integer specifications - Add octal integer specification - Zero (0) is considered an octal number - Add negative lookbehind/lookahead for number specifications (text is not word/token based) - Add method to evaluate constant expression for define constant expressions - Replace int conversion with constant evaluation expression - int conversion failed for hex numbers due to default base 10 [int(s)] vs unknown base [int(s, 0)] - Add additional tests --- SCons/cpp.py | 52 ++++++-- SCons/cppTests.py | 297 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 332 insertions(+), 17 deletions(-) diff --git a/SCons/cpp.py b/SCons/cpp.py index 1093ae2ac2..63e49ae88c 100644 --- a/SCons/cpp.py +++ b/SCons/cpp.py @@ -146,13 +146,27 @@ def Cleanup_CPP_Expressions(ts): # ...and compile the expression. CPP_to_Python_Ops_Expression = re.compile(expr) +# integer specifications +int_suffix_opt = r'(?:[uU](?:l{1,2}|L{1,2}|[zZ]|wb|WB)?|(?:l{1,2}|L{1,2}|[zZ]|wb|WB)[uU]?)?' + +hex_integer = fr'(0[xX][0-9A-Fa-f]+){int_suffix_opt}' +bin_integer = fr'(0[bB][01]+){int_suffix_opt}' +oct_integer = fr'(0[0-7]*){int_suffix_opt}' +dec_integer = fr'([1-9][0-9]*){int_suffix_opt}' + +int_boundary = r'[a-zA-Z0-9_]' +int_neg_lookbehind = fr'(? None: def scons_current_file(self, t) -> None: self.current_file = t[1] + def eval_constant_expression(self, s): + """ + Evaluates a C preprocessor expression. + + This is done by converting it to a Python equivalent and + eval()ing it in the C preprocessor namespace we use to + track #define values. + + Returns None if the eval() result is not an integer. + """ + s = CPP_to_Python(s) + try: + rval = eval(s, self.constant_expression_namespace) + except (NameError, TypeError, SyntaxError) as e: + rval = None + if not isinstance(rval, int): + rval = None + return rval + def eval_expression(self, t): """ Evaluates a C preprocessor expression. @@ -507,9 +543,11 @@ def do_define(self, t) -> None: Default handling of a #define line. """ _, name, args, expansion = t - try: - expansion = int(expansion) - except (TypeError, ValueError): + + rval = self.eval_constant_expression(expansion) + if rval is not None: + expansion = rval + else: # handle "defined" chain "! (defined (A) || defined (B)" ... if "defined " in expansion: self.cpp_namespace[name] = self.eval_expression(t[2:]) diff --git a/SCons/cppTests.py b/SCons/cppTests.py index 85f01b7813..8b079ec742 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -151,7 +151,7 @@ #include #endif -#if ! (defined (DEFINED_A) || defined (DEFINED_B) +#if ! (defined (DEFINED_A) || defined (DEFINED_B)) #include #else #include @@ -236,11 +236,189 @@ #include #endif -#if 123456789UL || 0x13L -#include +#if (123456789UL || 0x13L) +#include #else -#include +#include #endif + +#if (123456789UL && 0x0LU) +#include +#else +#include +#endif + +#if 123456789UL +#include +#else +#include +#endif + +#if 1234U +#include +#else +#include +#endif + +#if 1234L +#include +#else +#include +#endif + +#if 1234ULL +#include +#else +#include +#endif + +#define X1234UL 1 +#if X1234UL +#include +#else +#include +#endif + +#define X1234U 1 +#if X1234U +#include +#else +#include +#endif + +#define X1234L 1 +#if X1234L +#include +#else +#include +#endif + +#define X1234ULL 1 +#if X1234ULL +#include +#else +#include +#endif + +#define DEC0 0 +#define HEX0 0x0 +#define HEXF 0xF + +#if DEC0 +#include +#else +#include +#endif + +#if ! DEC0 +#include +#else +#include +#endif + +#if (DEC0) +#include +#else +#include +#endif + +#if !(DEC0) +#include +#else +#include +#endif + +#if HEX0 +#include +#else +#include +#endif + +#if ! HEX0 +#include +#else +#include +#endif + +#if (HEX0) +#include +#else +#include +#endif + +#if !(HEX0) +#include +#else +#include +#endif + +#if HEXF +#include +#else +#include +#endif + +#if ! HEXF +#include +#else +#include +#endif + +#if (HEXF) +#include +#else +#include +#endif + +#if !(HEXF) +#include +#else +#include +#endif + +#if defined(DEC0) && (DEC0 & 0x1) +#include +#else +#include +#endif + +#if !(defined(HEXF) && (HEXF & 0x1)) +#include +#else +#include +#endif + +#define X2345ULL 1 +#if !(X2345ULL > 4567ull) +#include +#else +#include +#endif + +#if !0ull +#include +#else +#include +#endif + +#define X0U 0U +#if X0U +#include +#else +#include +#endif + +#define XF1 (0x0U & 0x1U) +#if XF1 +#include +#else +#include +#endif + +#define ABC00 0U +#define ABC01 1U +#define ABC_(a, b) ABC##a##b +#define ABC ABC_(ZERO, ZERO) """ @@ -303,8 +481,8 @@ #define FUNC39a(x0, y0) FILE39 #define FUNC40a(x0, y0) FILE40 -#define FUNC39b(x1, y2) FUNC39a(x1, y1) -#define FUNC40b(x1, y2) FUNC40a(x1, y1) +#define FUNC39b(x1, y1) FUNC39a(x1, y1) +#define FUNC40b(x1, y1) FUNC40a(x1, y1) #define FUNC39c(x2, y2) FUNC39b(x2, y2) #define FUNC40c(x2, y2) FUNC40b(x2, y2) @@ -480,7 +658,12 @@ def test_expression(self) -> None: """Test #if with arithmetic expressions""" expect = self.expression_expect result = self.cpp.process_contents(expression_input) - assert expect == result, (expect, result) + if expect != result: + for e,r in zip(expect, result): + if e != r: + print("ERROR->",end="") + print(f"{e}: {r}") + assert expect == result, f"\nexpect:{expect}\nresult:{result}" def test_undef(self) -> None: """Test #undef handling""" @@ -588,7 +771,41 @@ class PreProcessorTestCase(cppAllTestCase): ('include', '<', 'file28-yes'), ('include', '"', 'file29-yes'), ('include', '<', 'file30-yes'), - ('include', '<', 'file301-yes'), + + ('include', '<', 'file301or-yes'), + ('include', '<', 'file301and-no'), + ('include', '<', 'file301ul-yes'), + ('include', '<', 'file301u-yes'), + ('include', '<', 'file301l-yes'), + ('include', '<', 'file301ull-yes'), + + ('include', '<', 'file302-yes'), + ('include', '<', 'file303-yes'), + ('include', '<', 'file304-yes'), + ('include', '<', 'file305-yes'), + + ('include', '<', 'file401-no'), + ('include', '<', 'file402-yes'), + ('include', '<', 'file403-no'), + ('include', '<', 'file404-yes'), + + ('include', '<', 'file411-no'), + ('include', '<', 'file412-yes'), + ('include', '<', 'file413-no'), + ('include', '<', 'file414-yes'), + + ('include', '<', 'file421-yes'), + ('include', '<', 'file422-no'), + ('include', '<', 'file423-yes'), + ('include', '<', 'file424-no'), + + ('include', '<', 'file431-no'), + ('include', '<', 'file432-no'), + + ('include', '<', 'file501-yes'), + ('include', '<', 'file502-yes'), + ('include', '<', 'file503-no'), + ('include', '<', 'file504-no'), ] undef_expect = [ @@ -717,8 +934,68 @@ class DumbPreProcessorTestCase(cppAllTestCase): ('include', '"', 'file29-yes'), ('include', '<', 'file30-yes'), ('include', '<', 'file30-no'), - ('include', '<', 'file301-yes'), - ('include', '<', 'file301-no'), + + ('include', '<', 'file301or-yes'), + ('include', '<', 'file301or-no'), + ('include', '<', 'file301and-yes'), + ('include', '<', 'file301and-no'), + ('include', '<', 'file301ul-yes'), + ('include', '<', 'file301ul-no'), + ('include', '<', 'file301u-yes'), + ('include', '<', 'file301u-no'), + ('include', '<', 'file301l-yes'), + ('include', '<', 'file301l-no'), + ('include', '<', 'file301ull-yes'), + ('include', '<', 'file301ull-no'), + + ('include', '<', 'file302-yes'), + ('include', '<', 'file302-no'), + ('include', '<', 'file303-yes'), + ('include', '<', 'file303-no'), + ('include', '<', 'file304-yes'), + ('include', '<', 'file304-no'), + ('include', '<', 'file305-yes'), + ('include', '<', 'file305-no'), + + ('include', '<', 'file401-yes'), + ('include', '<', 'file401-no'), + ('include', '<', 'file402-yes'), + ('include', '<', 'file402-no'), + ('include', '<', 'file403-yes'), + ('include', '<', 'file403-no'), + ('include', '<', 'file404-yes'), + ('include', '<', 'file404-no'), + + ('include', '<', 'file411-yes'), + ('include', '<', 'file411-no'), + ('include', '<', 'file412-yes'), + ('include', '<', 'file412-no'), + ('include', '<', 'file413-yes'), + ('include', '<', 'file413-no'), + ('include', '<', 'file414-yes'), + ('include', '<', 'file414-no'), + + ('include', '<', 'file421-yes'), + ('include', '<', 'file421-no'), + ('include', '<', 'file422-yes'), + ('include', '<', 'file422-no'), + ('include', '<', 'file423-yes'), + ('include', '<', 'file423-no'), + ('include', '<', 'file424-yes'), + ('include', '<', 'file424-no'), + ('include', '<', 'file431-yes'), + ('include', '<', 'file431-no'), + ('include', '<', 'file432-yes'), + ('include', '<', 'file432-no'), + + ('include', '<', 'file501-yes'), + ('include', '<', 'file501-no'), + ('include', '<', 'file502-yes'), + ('include', '<', 'file502-no'), + ('include', '<', 'file503-yes'), + ('include', '<', 'file503-no'), + ('include', '<', 'file504-yes'), + ('include', '<', 'file504-no'), ] undef_expect = [ From e96cf6850a2f4b8ba9c6d3f26e3d92826c6eeb98 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 10 Nov 2024 20:18:46 -0500 Subject: [PATCH 2/2] Update CHANGES.txt and RELEASE.txt --- CHANGES.txt | 16 ++++++++++++++++ RELEASE.txt | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 098bba3c9b..b75f8560a7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -59,6 +59,22 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER and nodes from the MSVSProject return values. Resolves #4613. - MSVS: Remove the platform specification (i.e., platform = 'win32') from select test script environments. The platform specification appears superfluous. + - SCons C preprocessor changes: + - Preserve literals that contain valid integer substring specifications. + Previously, the integer suffix could be stripped from a symbol that contained + an integer and suffix substring. + - Update the optional integer suffixes to include the z|Z and wb|WB suffixes. + - Update the optional integer suffixes to include support for alternate + orderings of unsigned with long or long long as defined in the c/cpp + grammar. + - Update the optional integer suffixes for case insensitive specifications as + defined in the c/cpp grammar. + - Add support for binary integer constants. + - Add support for octal integer constants. Previously, octal integers were + evaluated as decimal integers. A literal zero (0) is treated as an octal + number. + - Change the attempted conversion of a define expansion from using int() to + a constant expression evaluation. From Alex James: - On Darwin, PermissionErrors are now handled while trying to access diff --git a/RELEASE.txt b/RELEASE.txt index 6ba7fc06f7..2cb6638066 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -56,12 +56,23 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY in the projects argument list rather than manually removing solution file names and nodes from the MSVSProject return values. +- SCons C preprocessor: + - Update the optional integer suffixes to include the z|Z and wb|WB + suffixes. + - Add support for binary integer constants. + - Add support for octal integer constants. Previously, octal integers + were evaluated as decimal integers. A literal zero (0) is treated as an + octal number. + - Change the method for attempted conversion of a define expansion value + to an integer from a literal to a constant expression evaluation. + - Add a tag to each CacheDir to let systems ignore backing it up (per https://bford.info/cachedir/). Update the way a CacheDir is created, since it now has to create two files. FIXES ----- + - PackageVariable now does what the documentation always said it does if the variable is used on the command line with one of the enabling string as the value: the variable's default value is produced (previously @@ -104,6 +115,17 @@ FIXES solution file. User-specified project GUIDs should now be correctly written to the solution file. +- SCons C preprocessor: Preserve literals that contain valid integer + substring specifications. Previously, the integer suffix could be + stripped from a symbol that contained an integer and suffix substring. + +- SCons C preprocessor: Update the optional integer suffixes to include + support for the alternate orderings of unsigned with long or long long as + defined in the c/cpp grammar. + +- SCons C preprocessor: Update the optional integer suffixes for case + insensitive specifications as defined in the c/cpp grammar. + - Fix nasm test for missing include file, cleanup. - Skip running a few validation tests if the user is root and the test is