Support custom cache locations with use_unsafe_shared_cache and COURS…

…IER_CACHE (#316)

* Support custom cache locations with use_unsafe_shared_cache and COURSIER_CACHE

* Update README.md

.bazelci/presubmit.yml

@@ -1,6 +1,10 @@
 ---
 tasks:
   ubuntu1604:
+    environment:
+      # This tests custom cache locations.
+      # https://github.com/bazelbuild/rules_jvm_external/pull/316
+      COURSIER_CACHE: /tmp/custom_coursier_cache
     shell_commands:
       - bazel run @unpinned_regression_testing//:pin
       - bazel run @unpinned_maven_install_in_custom_location//:pin
@@ -52,6 +56,10 @@ tasks:
   ubuntu1804_latest:
     platform: ubuntu1804
     bazel: latest
+    environment:
+      # This tests custom cache locations.
+      # https://github.com/bazelbuild/rules_jvm_external/pull/316
+      COURSIER_CACHE: /tmp/custom_coursier_cache
     shell_commands:
       - bazel run @unpinned_regression_testing//:pin
       - bazel run @unpinned_maven_install_in_custom_location//:pin
@@ -62,13 +70,21 @@ tasks:
       # These tests are currently incompatible with OpenJDK 11.
       - "-//tests/integration:UnsafeSharedCacheTest"
   macos:
+    environment:
+      # This tests custom cache locations.
+      # https://github.com/bazelbuild/rules_jvm_external/pull/316
+      COURSIER_CACHE: /tmp/custom_coursier_cache
     shell_commands:
       - bazel run @unpinned_regression_testing//:pin
       - bazel run @unpinned_maven_install_in_custom_location//:pin
       - bazel run @duplicate_artifacts_test//:pin
     test_targets:
       - "//..."
   windows:
+    environment:
+      # This tests custom cache locations.
+      # https://github.com/bazelbuild/rules_jvm_external/pull/316
+      COURSIER_CACHE: /tmp/custom_coursier_cache
     shell_commands:
       - bazel run @unpinned_regression_testing//:pin
       - bazel run @unpinned_maven_install_in_custom_location//:pin

README.md

@@ -320,7 +320,7 @@ maven_install(
 > is a caching mechanism that was implemented before artifact pinning,
 > which uses Coursier's own persistent cache. With artifact pinning and
 > maven_install.json, the persistent cache is integrated directly into
-> Bazel's own cache.
+> Bazel's internal cache.
 
 To download artifacts into a shared and persistent directory in your home
 directory, set `use_unsafe_shared_cache = True` in `maven_install`.
@@ -341,9 +341,25 @@ This is **not safe** as Bazel is currently not able to detect changes in the
 shared cache. For example, if an artifact is deleted from the shared cache,
 Bazel will not re-run the repository rule automatically.
 
+To change the location of the cache from the home directory, set the
+`COURSIER_CACHE` environment variable. You can also use the `--repo_env` flag to
+set the variable on the command line and in `.bazelrc` files:
+
+```
+$ bazel build @maven_with_unsafe_shared_cache//... --repo_env=COURSIER_CACHE=/tmp/custom_cache
+```
+
+This feature also enables checking the downloaded artifacts into your source
+tree by declaring `COURSIER_CACHE` to be `<project root>/some/directory`. For
+example:
+
+```
+$ bazel build @maven_with_unsafe_shared_cache//... --repo_env=COURSIER_CACHE=$(pwd)/third_party
+```
+
 The default value of `use_unsafe_shared_cache` is `False`. This means that Bazel
 will create independent caches for each `maven_install` repository, located at
-`$(bazel info output_base)/external/@repository_name/v1`.
+`$(bazel info output_base)/external/@<repository_name>/v1`.
 
 ### `artifact` helper macro
 

coursier.bzl

@@ -59,13 +59,14 @@ def _relativize_and_symlink_file(repository_ctx, absolute_path):
     #
     # We assume that coursier uses the default cache location
     # TODO(jin): allow custom cache locations
-    absolute_path_parts = absolute_path.split("v1/")
+    absolute_path_parts = absolute_path.split(get_coursier_cache_or_default(
+        repository_ctx.os.environ, repository_ctx.attr.use_unsafe_shared_cache))
     if len(absolute_path_parts) != 2:
         fail("Error while trying to parse the path of file in the coursier cache: " + absolute_path)
     else:
         # Make a symlink from the absolute path of the artifact to the relative
         # path within the output_base/external.
-        artifact_relative_path = "v1/" + absolute_path_parts[1]
+        artifact_relative_path = "v1" + absolute_path_parts[1]
         repository_ctx.symlink(absolute_path, repository_ctx.path(artifact_relative_path))
     return artifact_relative_path
 
@@ -421,6 +422,19 @@ def _deduplicate_artifacts(dep_tree):
     dep_tree.update({"dependencies": deduped_artifacts.values()})
     return dep_tree
 
+# Get the path to the cache directory containing Coursier-downloaded artifacts.
+#
+# This method is public for testing.
+def get_coursier_cache_or_default(env_dict, use_unsafe_shared_cache):
+    location = env_dict.get("COURSIER_CACHE")
+    # If we're not using the unsafe shared cache, or if COURSIER_CACHE is not defined,
+    # use 'external/<this repo>/v1/'. 'v1/' is the current version of the Coursier cache.
+    if not use_unsafe_shared_cache or location == None:
+        return "v1"
+    else:
+        # This is an absolute path.
+        return location
+
 def _coursier_fetch_impl(repository_ctx):
     # Not using maven_install.json, so we resolve and fetch from scratch.
     # This takes significantly longer as it doesn't rely on any local
@@ -497,8 +511,7 @@ def _coursier_fetch_impl(repository_ctx):
             cmd.extend(["--credentials", utils.repo_credentials(repository)])
     for a in excluded_artifacts:
         cmd.extend(["--exclude", ":".join([a["group"], a["artifact"]])])
-    if not repository_ctx.attr.use_unsafe_shared_cache:
-        cmd.extend(["--cache", "v1"])  # Download into $output_base/external/$maven_repo_name/v1
+
     if repository_ctx.attr.fetch_sources:
         cmd.append("--sources")
         cmd.append("--default=true")
@@ -508,8 +521,19 @@ def _coursier_fetch_impl(repository_ctx):
         # happen on *nix.
         cmd.extend(["--parallel", "1"])
 
+    environment = {}
+    coursier_cache_location = get_coursier_cache_or_default(
+        repository_ctx.os.environ, repository_ctx.attr.use_unsafe_shared_cache)
+    if not repository_ctx.attr.use_unsafe_shared_cache:
+        cmd.extend(["--cache", coursier_cache_location])  # Download into $output_base/external/$maven_repo_name/v1
+        # If not using the shared cache and the user did not specify a COURSIER_CACHE, set the default
+        # value to prevent Coursier from writing into home directories.
+        # https://github.com/bazelbuild/rules_jvm_external/issues/301
+        # https://github.com/coursier/coursier/blob/1cbbf39b88ee88944a8d892789680cdb15be4714/modules/paths/src/main/java/coursier/paths/CoursierPaths.java#L29-L56
+        environment = {"COURSIER_CACHE": str(repository_ctx.path(coursier_cache_location))}
+
     repository_ctx.report_progress("Resolving and fetching the transitive closure of %s artifact(s).." % len(artifact_coordinates))
-    exec_result = repository_ctx.execute(cmd, timeout = repository_ctx.attr.resolve_timeout)
+    exec_result = repository_ctx.execute(cmd, timeout = repository_ctx.attr.resolve_timeout, environment = environment)
     if (exec_result.return_code != 0):
         fail("Error while fetching artifact with coursier: " + exec_result.stderr)
 
@@ -804,6 +828,7 @@ coursier_fetch = repository_rule(
         "HTTPS_PROXY",
         "no_proxy",
         "NO_PROXY",
+        "COURSIER_CACHE",
     ],
     implementation = _coursier_fetch_impl,
 )

tests/unit/coursier_test.bzl

@@ -4,6 +4,7 @@ load(
     "add_netrc_entries_from_mirror_urls",
     "extract_netrc_from_auth_url",
     "get_netrc_lines_from_entries",
+    "get_coursier_cache_or_default",
     "remove_auth_from_url",
     "split_url",
     infer = "infer_artifact_path_from_primary_and_repos",
@@ -356,6 +357,52 @@ def _get_netrc_lines_from_entries_multi_test_impl(ctx):
 
 get_netrc_lines_from_entries_multi_test = add_test(_get_netrc_lines_from_entries_multi_test_impl)
 
+def _mock_repo_path(path):
+    if path.startswith("/"):
+        return path
+    else:
+        return "/mockroot/" + path
+
+def _get_coursier_cache_or_default_disabled_test(ctx):
+    env = unittest.begin(ctx)
+    mock_environ = {
+        "COURSIER_CACHE": _mock_repo_path("/does/not/matter")
+    }
+    asserts.equals(
+        env,
+        "v1",
+        get_coursier_cache_or_default(mock_environ, False)
+    )
+    return unittest.end(env)
+
+get_coursier_cache_or_default_disabled_test = add_test(_get_coursier_cache_or_default_disabled_test)
+
+def _get_coursier_cache_or_default_enabled_with_default_location_test(ctx):
+    env = unittest.begin(ctx)
+    mock_environ = {}
+    asserts.equals(
+        env,
+        "v1",
+        get_coursier_cache_or_default(mock_environ, True)
+    )
+    return unittest.end(env)
+
+get_coursier_cache_or_default_enabled_with_default_location_test = add_test(_get_coursier_cache_or_default_enabled_with_default_location_test)
+
+def _get_coursier_cache_or_default_enabled_with_custom_location_test(ctx):
+    env = unittest.begin(ctx)
+    mock_environ = {
+        "COURSIER_CACHE": _mock_repo_path("/custom/location")
+    }
+    asserts.equals(
+        env,
+        "/custom/location",
+        get_coursier_cache_or_default(mock_environ, True)
+    )
+    return unittest.end(env)
+
+get_coursier_cache_or_default_enabled_with_custom_location_test = add_test(_get_coursier_cache_or_default_enabled_with_custom_location_test)
+
 def coursier_test_suite():
     unittest.suite(
         "coursier_tests",