Skip to content

Commit

Permalink
Merge pull request #81 from robotpy/extern-c
Browse files Browse the repository at this point in the history 
Fix extern C and add option to remove preprocessing
  • Loading branch information
@virtuald
virtuald authored Dec 5, 2022
2 parents e366144 + da1a074 commit 3980c3d
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 55 deletions.
111 changes: 56 additions & 55 deletions CppHeaderParser/CppHeaderParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,9 @@ def trace_print(*args):
#: Symbols to ignore, usually special macros
ignoreSymbols = ["Q_OBJECT"]

_BRACE_REASON_OTHER = 0
_BRACE_REASON_NS = 1
_BRACE_REASON_EXTERN = 2

# Track what was added in what order and at what depth
parseHistory = []
Expand Down Expand Up @@ -1477,7 +1480,6 @@ class Resolver(object):
C_KEYWORDS = "extern virtual static explicit inline friend constexpr".split()
C_KEYWORDS = set(C_KEYWORDS)

SubTypedefs = {} # TODO deprecate?
NAMESPACES = []
CLASSES = {}

Expand Down Expand Up @@ -1506,6 +1508,11 @@ def cur_namespace(self, add_double_colon=False):
i += 1
return rtn

def cur_linkage(self):
if len(self.linkage_stack):
return self.linkage_stack[-1]
return ""

def guess_ctypes_type(self, string):
pointers = string.count("*")
string = string.replace("*", "")
Expand Down Expand Up @@ -1870,21 +1877,6 @@ def finalize_vars(self):
var["ctypes_type"] = "ctypes.c_void_p"
var["unresolved"] = True

elif tag in self.SubTypedefs: # TODO remove SubTypedefs
if (
"property_of_class" in var
or "property_of_struct" in var
):
trace_print(
"class:", self.SubTypedefs[tag], "tag:", tag
)
var["typedef"] = self.SubTypedefs[tag] # class name
var["ctypes_type"] = "ctypes.c_void_p"
else:
trace_print("WARN-this should almost never happen!")
trace_print(var)
var["unresolved"] = True

elif tag in self._template_typenames:
var["typename"] = tag
var["ctypes_type"] = "ctypes.c_void_p"
Expand Down Expand Up @@ -2061,10 +2053,6 @@ def finalize(self):
trace_print("meth returns class:", meth["returns"])
meth["returns_class"] = True

elif meth["returns"] in self.SubTypedefs:
meth["returns_class"] = True
meth["returns_nested"] = self.SubTypedefs[meth["returns"]]

elif meth["returns"] in cls._public_enums:
enum = cls._public_enums[meth["returns"]]
meth["returns_enum"] = enum.get("type")
Expand Down Expand Up @@ -2372,6 +2360,7 @@ def _evaluate_method_stack(self):
self._get_location(self.nameStack),
)
newMethod["parent"] = None
newMethod["linkage"] = self.cur_linkage()
self.functions.append(newMethod)

# Reset template once it has been used
Expand Down Expand Up @@ -2472,7 +2461,6 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
klass["typedefs"][self.curAccessSpecifier].append(name)
if self.curAccessSpecifier == "public":
klass._public_typedefs[name] = typedef["type"]
Resolver.SubTypedefs[name] = self.curClass
else:
assert 0
elif self.curClass:
Expand Down Expand Up @@ -2524,6 +2512,7 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
self._get_location(self.nameStack),
)
newVar["namespace"] = self.current_namespace()
newVar["linkage"] = self.cur_linkage()
if self.curClass:
klass = self.classes[self.curClass]
klass["properties"][self.curAccessSpecifier].append(newVar)
Expand All @@ -2542,6 +2531,7 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None):
self._get_location(self.nameStack),
)
newVar["namespace"] = self.cur_namespace(False)
newVar["linkage"] = self.cur_linkage()
if addToVar:
newVar.update(addToVar)
self.variables.append(newVar)
Expand Down Expand Up @@ -2600,6 +2590,7 @@ def _evaluate_class_stack(self):
)
self.curTemplate = None
newClass["declaration_method"] = self.nameStack[0]
newClass["linkage"] = self.cur_linkage()
self.classes_order.append(newClass) # good idea to save ordering
self.stack = [] # fixes if class declared with ';' in closing brace
self.stmtTokens = []
Expand Down Expand Up @@ -2702,7 +2693,14 @@ def show(self):
for className in list(self.classes.keys()):
self.classes[className].show()

def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
def __init__(
self,
headerFileName,
argType="file",
encoding=None,
preprocessed=False,
**kwargs
):
"""Create the parsed C++ header file parse tree
headerFileName - Name of the file to parse OR actual file contents (depends on argType)
Expand Down Expand Up @@ -2775,6 +2773,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
self.curAccessSpecifier = "private" # private is default
self.curTemplate = None
self.accessSpecifierStack = []
self.linkage_stack = []
debug_print(
"curAccessSpecifier changed/defaulted to %s", self.curAccessSpecifier
)
Expand Down Expand Up @@ -2802,31 +2801,24 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
"[ ]+", " ", supportedAccessSpecifier[i]
).strip()

# Change multi line #defines and expressions to single lines maintaining line nubmers
# Based from http://stackoverflow.com/questions/2424458/regular-expression-to-match-cs-multiline-preprocessor-statements
matches = re.findall(r"(?m)^(?:.*\\\r?\n)+.*$", headerFileStr)
is_define = re.compile(r"[ \t\v]*#[Dd][Ee][Ff][Ii][Nn][Ee]")
for m in matches:
# Keep the newlines so that linecount doesnt break
num_newlines = len([a for a in m if a == "\n"])
if is_define.match(m):
new_m = m.replace("\n", "<CppHeaderParser_newline_temp_replacement>\\n")
else:
# Just expression taking up multiple lines, make it take 1 line for easier parsing
new_m = m.replace("\\\n", " ")
if num_newlines > 0:
new_m += "\n" * (num_newlines)
headerFileStr = headerFileStr.replace(m, new_m)

# Filter out Extern "C" statements. These are order dependent
matches = re.findall(
re.compile(r'extern[\t ]+"[Cc]"[\t \n\r]*{', re.DOTALL), headerFileStr
)
for m in matches:
# Keep the newlines so that linecount doesnt break
num_newlines = len([a for a in m if a == "\n"])
headerFileStr = headerFileStr.replace(m, "\n" * num_newlines)
headerFileStr = re.sub(r'extern[ ]+"[Cc]"[ ]*', "", headerFileStr)
if not preprocessed:
# Change multi line #defines and expressions to single lines maintaining line nubmers
# Based from http://stackoverflow.com/questions/2424458/regular-expression-to-match-cs-multiline-preprocessor-statements
matches = re.findall(r"(?m)^(?:.*\\\r?\n)+.*$", headerFileStr)
is_define = re.compile(r"[ \t\v]*#[Dd][Ee][Ff][Ii][Nn][Ee]")
for m in matches:
# Keep the newlines so that linecount doesnt break
num_newlines = len([a for a in m if a == "\n"])
if is_define.match(m):
new_m = m.replace(
"\n", "<CppHeaderParser_newline_temp_replacement>\\n"
)
else:
# Just expression taking up multiple lines, make it take 1 line for easier parsing
new_m = m.replace("\\\n", " ")
if num_newlines > 0:
new_m += "\n" * (num_newlines)
headerFileStr = headerFileStr.replace(m, new_m)

# Filter out any ignore symbols that end with "()" to account for #define magic functions
for ignore in ignoreSymbols:
Expand Down Expand Up @@ -2866,6 +2858,8 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
)

self.braceDepth = 0
self.braceReason = []
self.lastBraceReason = _BRACE_REASON_OTHER

lex = Lexer(self.headerFileName)
lex.input(headerFileStr)
Expand Down Expand Up @@ -2938,23 +2932,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
continue

if parenDepth == 0 and tok.type == "{":
self.lastBraceReason = _BRACE_REASON_OTHER
if len(self.nameStack) >= 2 and is_namespace(
self.nameStack
): # namespace {} with no name used in boost, this sets default?
if (
self.nameStack[1]
== "__IGNORED_NAMESPACE__CppHeaderParser__"
): # Used in filtering extern "C"
self.nameStack[1] = ""
self.nameSpaces.append("".join(self.nameStack[1:]))
ns = self.cur_namespace()
self.stack = []
self.stmtTokens = []
if ns not in self.namespaces:
self.namespaces.append(ns)
self.lastBraceReason = _BRACE_REASON_NS
# Detect special condition of macro magic before class declaration so we
# can filter it out
if "class" in self.nameStack and self.nameStack[0] != "class":
elif "class" in self.nameStack and self.nameStack[0] != "class":
classLocationNS = self.nameStack.index("class")
classLocationS = self.stack.index("class")
if (
Expand Down Expand Up @@ -2987,14 +2978,20 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs):
self.stmtTokens = []
if not self.braceHandled:
self.braceDepth += 1
self.braceReason.append(self.lastBraceReason)

elif parenDepth == 0 and tok.type == "}":
if self.braceDepth == 0:
continue
if self.braceDepth == len(self.nameSpaces):
tmp = self.nameSpaces.pop()
reason = self.braceReason.pop()
if reason == _BRACE_REASON_NS:
self.nameSpaces.pop()
self.stack = [] # clear stack when namespace ends?
self.stmtTokens = []
elif reason == _BRACE_REASON_EXTERN:
self.linkage_stack.pop()
self.stack = [] # clear stack when linkage ends?
self.stmtTokens = []
else:
self._evaluate_stack()
self.braceDepth -= 1
Expand Down Expand Up @@ -3358,8 +3355,10 @@ def _evaluate_stack(self, token=None):
pass
elif len(self.nameStack) == 2 and self.nameStack[0] == "extern":
debug_print("trace extern")
self.linkage_stack.append(self.nameStack[1].strip('"'))
self.stack = []
self.stmtTokens = []
self.lastBraceReason = _BRACE_REASON_EXTERN
elif (
len(self.nameStack) == 2 and self.nameStack[0] == "friend"
): # friend class declaration
Expand Down Expand Up @@ -3667,12 +3666,14 @@ def _parse_enum(self):
def _install_enum(self, newEnum, instancesData):
if len(self.curClass):
newEnum["namespace"] = self.cur_namespace(False)
newEnum["linkage"] = self.cur_linkage()
klass = self.classes[self.curClass]
klass["enums"][self.curAccessSpecifier].append(newEnum)
if self.curAccessSpecifier == "public" and "name" in newEnum:
klass._public_enums[newEnum["name"]] = newEnum
else:
newEnum["namespace"] = self.cur_namespace(True)
newEnum["linkage"] = self.cur_linkage()
self.enums.append(newEnum)
if "name" in newEnum and newEnum["name"]:
self.global_enums[newEnum["name"]] = newEnum
Expand Down
46 changes: 46 additions & 0 deletions test/test_CppHeaderParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4183,5 +4183,51 @@ def test_fn3(self):
self.assertEqual(fn["pure_virtual"], True)


class ExternCQuirk(unittest.TestCase):
# bug where extern "C" reset the namespace
def setUp(self):
self.cppHeader = CppHeaderParser.CppHeader(
"""
namespace cs {
extern "C" {
struct InCSAndExtern {};
void FnInCSAndExtern(InCSAndExtern *n);
}
class InCS {};
}
void FnNotInCSOrExtern();
""",
"string",
)

def test_fn(self):

# NotCS should be in namespace cs, extern C
c = self.cppHeader.classes["InCSAndExtern"]
self.assertEqual(c["namespace"], "cs")
self.assertEqual(c["linkage"], "C")

# FnNotCS should be in namespace cs, extern C
fn = self.cppHeader.functions[0]
self.assertEqual(fn["name"], "FnInCSAndExtern")
self.assertEqual(fn["namespace"], "cs::")
self.assertEqual(fn["linkage"], "C")

# InCS should be in namespace cs
c = self.cppHeader.classes["InCS"]
self.assertEqual(c["namespace"], "cs")
self.assertEqual(c["linkage"], "")

# FnNotCS should not in namespace cs nor extern C
fn = self.cppHeader.functions[1]
self.assertEqual(fn["name"], "FnNotInCSOrExtern")
self.assertEqual(fn["namespace"], "")
self.assertEqual(fn["linkage"], "")


if __name__ == "__main__":
unittest.main()

0 comments on commit 3980c3d

Please sign in to comment.