Add GOTCHA support

README.md

@@ -1,6 +1,8 @@
 wrap.py
 ===========================
-a [PMPI](http://www.open-mpi.org/faq/?category=perftools#PMPI) wrapper generator
+
+A [PMPI](http://www.open-mpi.org/faq/?category=perftools#PMPI) or 
+[GOTCHA](https://github.com/LLNL/GOTCHA) wrapper generator for MPI.
 
 by Todd Gamblin, tgamblin@llnl.gov, https://github.com/tgamblin/wrap
 
@@ -9,6 +11,7 @@ by Todd Gamblin, tgamblin@llnl.gov, https://github.com/tgamblin/wrap
        the Argonne PMPI wrapper generator, with some enhancements.
      Options:"
        -d             Just dump function declarations parsed out of mpi.h
+       -G             Generate GOTCHA instead of PMPI wrappers.
        -f             Generate fortran wrappers in addition to C wrappers.
        -g             Generate reentry guards around wrapper functions.
        -c exe         Provide name of MPI compiler (for parsing mpi.h).
@@ -248,6 +251,68 @@ They're not designed for making wrappers, but declarations of lots of variables
 * `{{<varname>}}` *(not yet supported)*
 	Access a variable declared by `{{vardecl}}`.
 
+GOTCHA wrappers
+-------------------------------
+
+With the `-G` option, wrap.py generates GOTCHA instead of PMPI
+wrappers. [GOTCHA](https://github.com/LLNL/GOTCHA) is a library to
+wrap library functions at runtime. It is similar to LD_PRELOAD, but
+operates via a programmable API. This way, we can generate wrappers
+for all MPI functions, but decide at runtime which functions should
+actually be wrapped. The GOTCHA instrumentation mechanism also avoids
+certain linking pitfalls that can hamper with PMPI's weak symbol approach.
+
+GOTCHA works by binding target functions to wrapper functions at
+runtime. A pointer to the original target function is saved, and can
+be used to call the original function from inside the wrapper. In
+GOTCHA mode, wrap.py generates three things for each wrapped MPI
+function:
+
+* A pointer to the original function `wrap_MPI_Foo_orig`, initialized
+  to `NULL`.
+
+* The wrapper function `wrap_MPI_Foo`. A `{{callfn}}` call inside the
+  body will call the original function through the saved function
+  pointer in `wrap_MPI_Foo_orig`.
+
+* A GOTCHA binding struct `wrap_MPI_Foo_binding` of type
+  `gotcha_binding_t`. This struct must be passed to `gotcha_wrap()` by
+  the target application or tool to activate the wrapper.
+
+Unlike in PMPI mode, generating the wrapper and linking it to the
+target application does *not* automatically wrap the MPI functions.
+The target application or tool must invoke `gotcha_wrap()` via some
+other instrumentation mechanism to activate the function wrappers.
+
+As an example, consider the following simple wrapper:
+
+    {{fn func MPI_Send}}{
+        printf("MPI function: %s\n", "{{func}}");
+        {{callfn}}
+    }{{endfn}}
+
+In GOTCHA mode, this will produce the following output:
+
+    /* ================== C Wrappers for MPI_Send ================== */
+    int (*wrap_MPI_Send_orig)(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) = NULL;
+
+    _EXTERN_C_ int wrap_MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { 
+        int _wrap_py_return_val = 0;
+    {
+        printf("MPI function: %s\n", "MPI_Send");
+        _wrap_py_return_val = (*wrap_MPI_Send_orig)(buf, count, datatype, dest, tag, comm);
+    }    return _wrap_py_return_val;
+    }
+
+    struct gotcha_binding_t wrap_MPI_Send_binding = { "MPI_Send", (void*) wrap_MPI_Send, &wrap_MPI_Send_orig };
+
+The target application must now activate the wrapper somewhere via
+`gotcha_wrap`:
+
+    gotcha_wrap(&wrap_MPI_Send_binding, 1, "my wrapper");
+
+Note that GOTCHA mode currently does not generate fortran wrappers.
+
 Notes on the fortran wrappers
 -------------------------------
     #if (!defined(MPICH_HAS_C2F) && defined(MPICH_NAME) && (MPICH_NAME == 1))

wrap.py

@@ -32,11 +32,12 @@
 #################################################################################################
 from __future__ import print_function
 usage_string = \
-'''Usage: wrap.py [-fgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...]
+'''Usage: wrap.py [-Gfgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...]
  Python script for creating PMPI wrappers. Roughly follows the syntax of
    the Argonne PMPI wrapper generator, with some enhancements.
  Options:"
    -d             Just dump function declarations parsed out of mpi.h
+   -G             Generate GOTCHA instead of PMPI wrappers
    -f             Generate fortran wrappers in addition to C wrappers.
    -g             Generate reentry guards around wrapper functions.
    -s             Skip writing #includes, #defines, and other front-matter (for non-C output).
@@ -64,6 +65,7 @@
 skip_headers = False               # Skip header information and defines (for non-C output)
 dump_prototypes = False            # Just exit and dump MPI protos if false.
 ignore_deprecated = False          # Do not print compiler warnings for deprecated MPI functions
+generate_gotcha = False            # Generate GOTCHA instead of PMPI wrappers
 
 # Possible legal bindings for the fortran version of PMPI_Init()
 pmpi_init_bindings = ["PMPI_INIT", "pmpi_init", "pmpi_init_", "pmpi_init__"]
@@ -134,6 +136,11 @@
 
 '''
 
+gotcha_wrapper_includes = ''' 
+#include <gotcha/gotcha.h>
+
+''' 
+
 # Macros used to suppress MPI deprecation warnings.
 wrapper_diagnosics_macros = '''
 /* Macros to enable and disable compiler warnings for deprecated functions.
@@ -519,6 +526,10 @@ def pmpi_prototype(self, modifiers=""):
         if modifiers: modifiers = joinlines(modifiers, " ")
         return "%s%s P%s(%s)" % (modifiers, self.retType(), self.name, ", ".join(self.formals()))
 
+    def gotcha_prototype(self, modifiers=""):
+        if modifiers: modifiers = joinlines(modifiers, " ")
+        return "%s%s wrap_%s(%s)" % (modifiers, self.retType(), self.name, ", ".join(self.formals()))
+
     def fortranPrototype(self, name=None, modifiers=""):
         if not name: name = self.name
         if modifiers: modifiers = joinlines(modifiers, " ")
@@ -630,8 +641,25 @@ def write_exit_guard(out):
     if output_guards:
         out.write("    in_wrapper = 0;\n")
 
+def write_gotcha_c_wrapper(out, decl, return_val, write_body):
+    """Write the C wrapper for an MPI function."""
+    # Write the pointer to the original function
+    out.write("%s (*wrap_%s_orig)(%s) = NULL;\n" % (decl.retType(), decl.name, ", ".join(decl.formals())))
+
+    # Now write the wrapper function, which will call the original function through the pointer
+    out.write(decl.gotcha_prototype(default_modifiers))
+    out.write(" { \n")
+    out.write("    %s %s = 0;\n" % (decl.retType(), return_val))
 
-def write_c_wrapper(out, decl, return_val, write_body):
+    write_body(out)
+
+    out.write("    return %s;\n" % return_val)
+    out.write("}\n\n")
+
+    # Write the GOTCHA binding struct
+    out.write("struct gotcha_binding_t wrap_%s_binding = { \"%s\", (void*) wrap_%s, &wrap_%s_orig };\n\n" % (decl.name, decl.name, decl.name, decl.name))
+
+def write_pmpi_c_wrapper(out, decl, return_val, write_body):
     """Write the C wrapper for an MPI function."""
     # Write the PMPI prototype here in case mpi.h doesn't define it
     # (sadly the case with some MPI implementaitons)
@@ -650,6 +678,12 @@ def write_c_wrapper(out, decl, return_val, write_body):
     out.write("    return %s;\n" % return_val)
     out.write("}\n\n")
 
+def write_c_wrapper(out, decl, return_val, write_body):
+    """Write the C wrapper for an MPI function."""
+    if generate_gotcha:
+        write_gotcha_c_wrapper(out, decl, return_val, write_body)
+    else:
+        write_pmpi_c_wrapper(out, decl, return_val, write_body)
 
 def write_fortran_binding(out, decl, delegate_name, binding, stmts=None):
     """Outputs a wrapper for a particular fortran binding that delegates to the
@@ -948,7 +982,9 @@ def fn(out, scope, args, children):
         fn_scope["ret_val"] = return_val
         fn_scope["returnVal"]  = fn_scope["ret_val"]  # deprecated name.
 
-        if ignore_deprecated:
+        if generate_gotcha:
+            c_call = "%s = (*wrap_%s_orig)(%s);" % (return_val, fn.name, ", ".join(fn.argNames()))
+        elif ignore_deprecated:
             c_call = "%s\n%s = P%s(%s);\n%s" % ("WRAP_MPI_CALL_PREFIX", return_val, fn.name, ", ".join(fn.argNames()), "WRAP_MPI_CALL_POSTFIX")
         else:
             c_call = "%s = P%s(%s);" % (return_val, fn.name, ", ".join(fn.argNames()))
@@ -1277,13 +1313,14 @@ def usage():
 output_filename = None
 
 try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "fsgdwc:o:i:I:")
+    opts, args = getopt.gnu_getopt(sys.argv[1:], "Gfsgdwc:o:i:I:")
 except getopt.GetoptError as err:
     sys.stderr.write(err + "\n")
     usage()
 
 for opt, arg in opts:
     if opt == "-d": dump_prototypes = True
+    if opt == "-G": generate_gotcha = True
     if opt == "-f": output_fortran_wrappers = True
     if opt == "-s": skip_headers = True
     if opt == "-g": output_guards = True
@@ -1303,6 +1340,10 @@ def usage():
 if len(args) < 1 and not dump_prototypes:
     usage()
 
+if generate_gotcha and output_fortran_wrappers:
+    sys.stderr.write("gotcha: Error: Cannot write fortran wrappers in GOTCHA mode")
+    sys.exit(1)
+
 # Parse mpi.h and put declarations into a map.
 for decl in enumerate_mpi_declarations(mpicc, includes):
     mpi_functions[decl.name] = decl
@@ -1328,6 +1369,8 @@ def usage():
     # Start with some headers and definitions.
     if not skip_headers:
         output.write(wrapper_includes)
+        if generate_gotcha:
+            output.write(gotcha_wrapper_includes)
         if output_guards: output.write("static int in_wrapper = 0;\n")
 
     # Print the macros for disabling MPI function deprecation warnings.