util/mr_cache: Support compile default monitor

--with-default-monitor can be used to define a default memory monitor
instead of relying on the hardcoded ordered list. Options are memhooks,
uffd, or disabled.

Signed-off-by: Ian Ziemba <ian.ziemba@hpe.com>

configure.ac

@@ -885,6 +885,25 @@ AC_ARG_ENABLE([uffd-monitor],
 AC_DEFINE_UNQUOTED(ENABLE_UFFD_MONITOR, [$enable_uffd],
 	[Define to 1 to enable uffd memory monitor])
 
+default_monitor=""
+bad_default="0"
+AC_ARG_WITH([default-monitor],
+	[AS_HELP_STRING([--with-default-monitor=<memhooks|uffd|disabled>],
+			[Select the default memory monitor.])],
+	[AS_CASE([$with_default_monitor],
+		[memhooks],[default_monitor=memhooks],
+		[uffd],[default_monitor=uffd],
+		[disabled], [default_monitor=disabled],
+		[AC_MSG_ERROR([Unknown monitor specified: $with_default_monitor.  Choices are memhooks, uffd, or disabled.])])
+	 AS_CASE([$default_monitor],
+		[memhooks], [AS_IF([test "$enable_memhooks" != "1"], [bad_default=1])],
+		[uffd], [AS_IF([test "$enable_uffd" != "1"], [bad_default=1])],
+		[])
+	 AS_IF([test "$bad_default" != "0"],
+		[AC_MSG_ERROR(["Default memory monitor is not available: $default_monitor."])])
+	 AC_DEFINE_UNQUOTED([HAVE_MR_CACHE_MONITOR_DEFAULT], ["$default_monitor"], [Default memory monitor])
+	],
+	[])
 
 AH_BOTTOM([
 #if defined(__linux__) && (defined(__x86_64__) || defined(__amd64__) || defined(__aarch64__) || (defined(__riscv) && __riscv_xlen == 64)) && ENABLE_MEMHOOKS_MONITOR

prov/util/src/util_mem_monitor.c

@@ -212,6 +212,30 @@ static void cleanup_monitor_list() {
 	monitor_list_size = 0;
 }
 
+static void set_default_monitor(const char *monitor)
+{
+	if (!monitor)
+		return;
+
+	if (!strcmp(monitor, "userfaultfd") || !strcmp(monitor, "uffd")) {
+#if HAVE_UFFD_MONITOR
+		default_monitor = uffd_monitor;
+#else
+		FI_WARN(&core_prov, FI_LOG_MR, "userfaultfd monitor not available\n");
+		default_monitor = NULL;
+#endif
+	} else if (!strcmp(monitor, "memhooks")) {
+#if HAVE_MEMHOOKS_MONITOR
+		default_monitor = memhooks_monitor;
+#else
+		FI_WARN(&core_prov, FI_LOG_MR, "memhooks monitor not available\n");
+		default_monitor = NULL;
+#endif
+	} else if (!strcmp(monitor, "disabled")) {
+		default_monitor = NULL;
+	}
+}
+
 /*
  * Initialize all available memory monitors
  */
@@ -248,9 +272,14 @@ void ofi_monitors_init(void)
 			" address changes.  Options are: userfaultfd, memhooks"
 			" and disabled.  Userfaultfd is a Linux kernel feature."
 			" Memhooks operates by intercepting memory allocation"
-			" and free calls.  Userfaultfd is the default if"
-			" available on the system. 'disabled' option disables"
-			" memory caching.");
+			" and free calls."
+#if defined(HAVE_MR_CACHE_MONITOR_DEFAULT)
+			" " HAVE_MR_CACHE_MONITOR_DEFAULT
+#else
+			" Userfaultfd"
+#endif
+			" is the default if available on the system. 'disabled'"
+			" option disables memory caching.");
 	fi_param_define(NULL, "mr_cuda_cache_monitor_enabled", FI_PARAM_BOOL,
 			"Enable or disable the CUDA cache memory monitor."
 			"Enabled by default.");
@@ -279,7 +308,9 @@ void ofi_monitors_init(void)
 	 * do not override
 	 */
 	if (!default_monitor) {
-#if HAVE_MEMHOOKS_MONITOR
+#if defined(HAVE_MR_CACHE_MONITOR_DEFAULT)
+		set_default_monitor(HAVE_MR_CACHE_MONITOR_DEFAULT);
+#elif HAVE_MEMHOOKS_MONITOR
 		default_monitor = memhooks_monitor;
 #elif HAVE_UFFD_MONITOR
 		default_monitor = uffd_monitor;
@@ -288,25 +319,8 @@ void ofi_monitors_init(void)
 #endif
 	}
 
-	if (cache_params.monitor != NULL) {
-		if (!strcmp(cache_params.monitor, "userfaultfd")) {
-#if HAVE_UFFD_MONITOR
-			default_monitor = uffd_monitor;
-#else
-			FI_WARN(&core_prov, FI_LOG_MR, "userfaultfd monitor not available\n");
-			default_monitor = NULL;
-#endif
-		} else if (!strcmp(cache_params.monitor, "memhooks")) {
-#if HAVE_MEMHOOKS_MONITOR
-			default_monitor = memhooks_monitor;
-#else
-			FI_WARN(&core_prov, FI_LOG_MR, "memhooks monitor not available\n");
-			default_monitor = NULL;
-#endif
-		} else if (!strcmp(cache_params.monitor, "disabled")) {
-			default_monitor = NULL;
-		}
-	}
+	if (cache_params.monitor != NULL)
+		set_default_monitor(cache_params.monitor);
 
 	FI_INFO(&core_prov, FI_LOG_MR,
 		"Default memory monitor is: %s\n",