From 625b7cc88172a37d26c53a55d9a0137e8c680935 Mon Sep 17 00:00:00 2001 From: David Boehme Date: Mon, 4 Dec 2017 12:28:11 -0800 Subject: [PATCH 1/4] Add GOTCHA support in wrap.py --- wrap.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/wrap.py b/wrap.py index 28a8c8a..0a9accf 100755 --- a/wrap.py +++ b/wrap.py @@ -32,11 +32,12 @@ ################################################################################################# from __future__ import print_function usage_string = \ -'''Usage: wrap.py [-fgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...] +'''Usage: wrap.py [-Gfgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...] Python script for creating PMPI wrappers. Roughly follows the syntax of the Argonne PMPI wrapper generator, with some enhancements. Options:" -d Just dump function declarations parsed out of mpi.h + -G Generate GOTCHA instead of PMPI wrappers -f Generate fortran wrappers in addition to C wrappers. -g Generate reentry guards around wrapper functions. -s Skip writing #includes, #defines, and other front-matter (for non-C output). @@ -64,6 +65,7 @@ skip_headers = False # Skip header information and defines (for non-C output) dump_prototypes = False # Just exit and dump MPI protos if false. ignore_deprecated = False # Do not print compiler warnings for deprecated MPI functions +generate_gotcha = False # Generate GOTCHA instead of PMPI wrappers # Possible legal bindings for the fortran version of PMPI_Init() pmpi_init_bindings = ["PMPI_INIT", "pmpi_init", "pmpi_init_", "pmpi_init__"] @@ -134,6 +136,11 @@ ''' +gotcha_wrapper_includes = ''' +#include + +''' + # Macros used to suppress MPI deprecation warnings. wrapper_diagnosics_macros = ''' /* Macros to enable and disable compiler warnings for deprecated functions. @@ -519,6 +526,10 @@ def pmpi_prototype(self, modifiers=""): if modifiers: modifiers = joinlines(modifiers, " ") return "%s%s P%s(%s)" % (modifiers, self.retType(), self.name, ", ".join(self.formals())) + def gotcha_prototype(self, modifiers=""): + if modifiers: modifiers = joinlines(modifiers, " ") + return "%s%s wrap_%s(%s)" % (modifiers, self.retType(), self.name, ", ".join(self.formals())) + def fortranPrototype(self, name=None, modifiers=""): if not name: name = self.name if modifiers: modifiers = joinlines(modifiers, " ") @@ -630,8 +641,25 @@ def write_exit_guard(out): if output_guards: out.write(" in_wrapper = 0;\n") +def write_gotcha_c_wrapper(out, decl, return_val, write_body): + """Write the C wrapper for an MPI function.""" + # Write the pointer to the original function + out.write("%s (*wrap_%s_orig)(%s) = NULL;\n" % (decl.retType(), decl.name, ", ".join(decl.types()))) + + # Now write the wrapper function, which will call the original function through the pointer + out.write(decl.gotcha_prototype(default_modifiers)) + out.write(" { \n") + out.write(" %s %s = 0;\n" % (decl.retType(), return_val)) -def write_c_wrapper(out, decl, return_val, write_body): + write_body(out) + + out.write(" return %s;\n" % return_val) + out.write("}\n\n") + + # Write the GOTCHA binding struct + out.write("struct gotcha_binding_t wrap_%s_binding = { \"%s\", (void*) wrap_%s, &wrap_%s_orig };\n\n" % (decl.name, decl.name, decl.name, decl.name)) + +def write_pmpi_c_wrapper(out, decl, return_val, write_body): """Write the C wrapper for an MPI function.""" # Write the PMPI prototype here in case mpi.h doesn't define it # (sadly the case with some MPI implementaitons) @@ -650,6 +678,12 @@ def write_c_wrapper(out, decl, return_val, write_body): out.write(" return %s;\n" % return_val) out.write("}\n\n") +def write_c_wrapper(out, decl, return_val, write_body): + """Write the C wrapper for an MPI function.""" + if generate_gotcha: + write_gotcha_c_wrapper(out, decl, return_val, write_body) + else: + write_pmpi_c_wrapper(out, decl, return_val, write_body) def write_fortran_binding(out, decl, delegate_name, binding, stmts=None): """Outputs a wrapper for a particular fortran binding that delegates to the @@ -948,7 +982,9 @@ def fn(out, scope, args, children): fn_scope["ret_val"] = return_val fn_scope["returnVal"] = fn_scope["ret_val"] # deprecated name. - if ignore_deprecated: + if generate_gotcha: + c_call = "%s = (*wrap_%s_orig)(%s);" % (return_val, fn.name, ", ".join(fn.argNames())) + elif ignore_deprecated: c_call = "%s\n%s = P%s(%s);\n%s" % ("WRAP_MPI_CALL_PREFIX", return_val, fn.name, ", ".join(fn.argNames()), "WRAP_MPI_CALL_POSTFIX") else: c_call = "%s = P%s(%s);" % (return_val, fn.name, ", ".join(fn.argNames())) @@ -1277,13 +1313,14 @@ def usage(): output_filename = None try: - opts, args = getopt.gnu_getopt(sys.argv[1:], "fsgdwc:o:i:I:") + opts, args = getopt.gnu_getopt(sys.argv[1:], "Gfsgdwc:o:i:I:") except getopt.GetoptError as err: sys.stderr.write(err + "\n") usage() for opt, arg in opts: if opt == "-d": dump_prototypes = True + if opt == "-G": generate_gotcha = True if opt == "-f": output_fortran_wrappers = True if opt == "-s": skip_headers = True if opt == "-g": output_guards = True @@ -1303,6 +1340,10 @@ def usage(): if len(args) < 1 and not dump_prototypes: usage() +if generate_gotcha and output_fortran_wrappers: + sys.stderr.write("gotcha: Error: Cannot write fortran wrappers in GOTCHA mode") + sys.exit(1) + # Parse mpi.h and put declarations into a map. for decl in enumerate_mpi_declarations(mpicc, includes): mpi_functions[decl.name] = decl @@ -1328,6 +1369,8 @@ def usage(): # Start with some headers and definitions. if not skip_headers: output.write(wrapper_includes) + if generate_gotcha: + output.write(gotcha_wrapper_includes) if output_guards: output.write("static int in_wrapper = 0;\n") # Print the macros for disabling MPI function deprecation warnings. From cc4d2b964fe860224bbed4c3b39cb00d3466593a Mon Sep 17 00:00:00 2001 From: David Boehme Date: Mon, 4 Dec 2017 14:56:17 -0800 Subject: [PATCH 2/4] Handle Pcontrol correctly in GOTCHA mode --- wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrap.py b/wrap.py index 0a9accf..73d3d6a 100755 --- a/wrap.py +++ b/wrap.py @@ -644,7 +644,7 @@ def write_exit_guard(out): def write_gotcha_c_wrapper(out, decl, return_val, write_body): """Write the C wrapper for an MPI function.""" # Write the pointer to the original function - out.write("%s (*wrap_%s_orig)(%s) = NULL;\n" % (decl.retType(), decl.name, ", ".join(decl.types()))) + out.write("%s (*wrap_%s_orig)(%s) = NULL;\n" % (decl.retType(), decl.name, ", ".join(decl.formals()))) # Now write the wrapper function, which will call the original function through the pointer out.write(decl.gotcha_prototype(default_modifiers)) From f2cb296b8c313f25e26b6ab9c53a2f1f57494420 Mon Sep 17 00:00:00 2001 From: David Boehme Date: Mon, 4 Dec 2017 14:56:54 -0800 Subject: [PATCH 3/4] Add documentation for GOTCHA mode --- README.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4c8c3b..f3348f8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ wrap.py =========================== -a [PMPI](http://www.open-mpi.org/faq/?category=perftools#PMPI) wrapper generator + +A [PMPI](http://www.open-mpi.org/faq/?category=perftools#PMPI) or +[GOTCHA](https://github.com/LLNL/GOTCHA) wrapper generator for MPI. by Todd Gamblin, tgamblin@llnl.gov, https://github.com/tgamblin/wrap @@ -9,6 +11,7 @@ by Todd Gamblin, tgamblin@llnl.gov, https://github.com/tgamblin/wrap the Argonne PMPI wrapper generator, with some enhancements. Options:" -d Just dump function declarations parsed out of mpi.h + -G Generate GOTCHA instead of PMPI wrappers. -f Generate fortran wrappers in addition to C wrappers. -g Generate reentry guards around wrapper functions. -c exe Provide name of MPI compiler (for parsing mpi.h). @@ -248,6 +251,66 @@ They're not designed for making wrappers, but declarations of lots of variables * `{{}}` *(not yet supported)* Access a variable declared by `{{vardecl}}`. +GOTCHA wrappers +------------------------------- + +With the `-G` option, wrap.py generates GOTCHA instead of PMPI +wrappers. [GOTCHA](https://github.com/LLNL/GOTCHA) is a library to +wrap library functions at runtime. It is similar to LD_PRELOAD, but +operates via a programmable API. This way, we can generate wrappers +for all MPI functions, but decide at runtime which functions should +actually be wrapped. The GOTCHA instrumentation mechanism also avoids +certain linking pitfalls that can hamper with PMPI's weak symbol approach. + +GOTCHA works by binding target functions to wrapper functions at +runtime. A pointer to the original target function is saved, and can +be used to call the original function from the wrapper. In GOTCHA +mode, wrap.py generates three things for each wrapped MPI function: + +* A pointer to the original function `wrap_MPI_Foo_orig`, initialized + to `NULL`. + +* The wrapper function `wrap_MPI_Foo`. The call to the original + function via `{{callfn}}` uses the function pointer. + +* A GOTCHA binding struct `wrap_MPI_Foo_binding` of type + `gotcha_binding_t` that must be passed to `gotcha_wrap()` by the + target application or tool to activate the wrapper. + +Unlike in PMPI mode, generating the wrapper and linking it to the +target application does *not* automatically wrap the MPI functions. +The target application or tool must invoke `gotcha_wrap()` via some +other instrumentation mechanism for all functions that should be +wrapped. + +As an example, consider the following simple wrapper: + + {{fn func MPI_Send}}{ + printf("MPI function: %s\n", "{{func}}"); + {{callfn}} + }{{endfn}} + +In GOTCHA mode, this will produce the following output: + + /* ================== C Wrappers for MPI_Send ================== */ + int (*wrap_MPI_Send_orig)(const void*, int, MPI_Datatype, int, int, MPI_Comm) = NULL; + _EXTERN_C_ int wrap_MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { + int _wrap_py_return_val = 0; + { + printf("MPI function: %s\n", "MPI_Send"); + _wrap_py_return_val = (*wrap_MPI_Send_orig)(buf, count, datatype, dest, tag, comm); + } return _wrap_py_return_val; + } + + struct gotcha_binding_t wrap_MPI_Send_binding = { "MPI_Send", (void*) wrap_MPI_Send, &wrap_MPI_Send_orig }; + +The target application must now activate the wrapper somewhere via +`gotcha_wrap`: + + gotcha_wrap(&wrap_MPI_Send_binding, 1, "my wrapper"); + +Note that GOTCHA mode currently does not generate fortran wrappers. + Notes on the fortran wrappers ------------------------------- #if (!defined(MPICH_HAS_C2F) && defined(MPICH_NAME) && (MPICH_NAME == 1)) From 7e99d7f08e15071daf016120f8a8565f74f911d4 Mon Sep 17 00:00:00 2001 From: David Boehme Date: Tue, 5 Dec 2017 08:32:21 -0800 Subject: [PATCH 4/4] GOTCHA doc updates --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f3348f8..28ce6e5 100644 --- a/README.md +++ b/README.md @@ -264,24 +264,25 @@ certain linking pitfalls that can hamper with PMPI's weak symbol approach. GOTCHA works by binding target functions to wrapper functions at runtime. A pointer to the original target function is saved, and can -be used to call the original function from the wrapper. In GOTCHA -mode, wrap.py generates three things for each wrapped MPI function: +be used to call the original function from inside the wrapper. In +GOTCHA mode, wrap.py generates three things for each wrapped MPI +function: * A pointer to the original function `wrap_MPI_Foo_orig`, initialized to `NULL`. -* The wrapper function `wrap_MPI_Foo`. The call to the original - function via `{{callfn}}` uses the function pointer. +* The wrapper function `wrap_MPI_Foo`. A `{{callfn}}` call inside the + body will call the original function through the saved function + pointer in `wrap_MPI_Foo_orig`. * A GOTCHA binding struct `wrap_MPI_Foo_binding` of type - `gotcha_binding_t` that must be passed to `gotcha_wrap()` by the - target application or tool to activate the wrapper. + `gotcha_binding_t`. This struct must be passed to `gotcha_wrap()` by + the target application or tool to activate the wrapper. Unlike in PMPI mode, generating the wrapper and linking it to the target application does *not* automatically wrap the MPI functions. The target application or tool must invoke `gotcha_wrap()` via some -other instrumentation mechanism for all functions that should be -wrapped. +other instrumentation mechanism to activate the function wrappers. As an example, consider the following simple wrapper: @@ -293,7 +294,8 @@ As an example, consider the following simple wrapper: In GOTCHA mode, this will produce the following output: /* ================== C Wrappers for MPI_Send ================== */ - int (*wrap_MPI_Send_orig)(const void*, int, MPI_Datatype, int, int, MPI_Comm) = NULL; + int (*wrap_MPI_Send_orig)(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) = NULL; + _EXTERN_C_ int wrap_MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { int _wrap_py_return_val = 0; {