diff --git a/coursier.bzl b/coursier.bzl index 05bf2e6de..5b124f8db 100644 --- a/coursier.bzl +++ b/coursier.bzl @@ -131,6 +131,72 @@ def _compute_dependency_tree_signature(artifacts): signature_inputs.append(":".join(artifact_group)) return hash(repr(sorted(signature_inputs))) +def extract_netrc_from_auth_url(url): + """Return a dict showing the netrc machine, login, and password extracted from a url. + + Returns: + A dict that is empty if there were no credentials in the url. + A dict that has three keys -- machine, login, password -- with their respective values. These values should be + what is needed for the netrc entry of the same name except for password whose value may be empty meaning that + there is no password for that login. + """ + if "@" not in url: + return {} + protocol, url_parts = split_url(url) + login_password_host = url_parts[0] + if "@" not in login_password_host: + return {} + login_password, host = login_password_host.rsplit("@", 1) + login_password_split = login_password.split(":", 1) + login = login_password_split[0] + # If password is not provided, then this will be a 1-length split + if len(login_password_split) < 2: + password = None + else: + password = login_password_split[1] + if not host: + fail("Got a blank host from: {}".format(url)) + if not login: + fail("Got a blank login from: {}".format(url)) + # Do not fail for blank password since that is sometimes a thing + return { + "machine": host, + "login": login, + "password": password, + } + +def add_netrc_entries_from_mirror_urls(netrc_entries, mirror_urls): + """Add a url's auth credentials into a netrc dict of form return[machine][login] = password.""" + for url in mirror_urls: + entry = extract_netrc_from_auth_url(url) + if not entry: + continue + machine = entry["machine"] + login = entry["login"] + password = entry["password"] + if machine not in netrc_entries: + netrc_entries[machine] = {} + if login not in netrc_entries[machine]: + if netrc_entries[machine]: + print("Received multiple logins for machine '{}'! Only using '{}'".format( + machine, netrc_entries[machine].keys()[0])) + continue + netrc_entries[machine][login] = password + else: + if netrc_entries[machine][login] != password: + print("Received different passwords for {}@{}! Only using the first".format(login, machine)) + return netrc_entries + +def get_netrc_lines_from_entries(netrc_entries): + netrc_lines = [] + for machine, login_dict in sorted(netrc_entries.items()): + for login, password in sorted(login_dict.items()): + netrc_lines.append("machine {}".format(machine)) + netrc_lines.append("login {}".format(login)) + if password: + netrc_lines.append("password {}".format(password)) + return netrc_lines + def _pinned_coursier_fetch_impl(repository_ctx): if not repository_ctx.attr.maven_install_json: fail("Please specify the file label to maven_install.json (e.g." + @@ -197,6 +263,7 @@ def _pinned_coursier_fetch_impl(repository_ctx): "load(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_file\")", "def pinned_maven_install():", ] + netrc_entries = {} for artifact in dep_tree["dependencies"]: if artifact.get("url") != None: http_file_repository_name = escape(artifact["coord"]) @@ -204,9 +271,15 @@ def _pinned_coursier_fetch_impl(repository_ctx): " http_file(", " name = \"%s\"," % http_file_repository_name, " sha256 = \"%s\"," % artifact["sha256"], + # repository_ctx should point to external/$repository_ctx.name + # The http_file should point to external/$http_file_repository_name + # File-path is relative defined from http_file traveling to repository_ctx. + " netrc = \"../%s/netrc\"," % (repository_ctx.name), ]) if artifact.get("mirror_urls") != None: - http_files.append(" urls = %s," % repr(artifact["mirror_urls"])) + http_files.append(" urls = %s," % repr( + [remove_auth_from_url(url) for url in artifact["mirror_urls"]])) + netrc_entries = add_netrc_entries_from_mirror_urls(netrc_entries, artifact["mirror_urls"]) else: # For backwards compatibility. mirror_urls is a field added in a # later version than the url field, so not all maven_install.json @@ -214,6 +287,7 @@ def _pinned_coursier_fetch_impl(repository_ctx): http_files.append(" urls = [\"%s\"]," % artifact["url"]) http_files.append(" )") repository_ctx.file("defs.bzl", "\n".join(http_files), executable = False) + repository_ctx.file("netrc", "\n".join(get_netrc_lines_from_entries(netrc_entries)), executable = False) repository_ctx.report_progress("Generating BUILD targets..") (generated_imports, jar_versionless_target_labels) = parser.generate_imports( @@ -272,13 +346,17 @@ def _pinned_coursier_fetch_impl(repository_ctx): False, # not executable ) +def split_url(url): + protocol = url[:url.find("://")] + url_without_protocol = url[url.find("://") + 3:] + url_parts = url_without_protocol.split("/") + return protocol, url_parts + def remove_auth_from_url(url): """Returns url without `user:pass@` or `user@`.""" if "@" not in url: return url - protocol = url[:url.find("://")] - url_without_protocol = url[url.find("://") + 3:] - url_parts = url_without_protocol.split("/") + protocol, url_parts = split_url(url) host = url_parts[0] if "@" not in host: return url diff --git a/tests/unit/coursier_test.bzl b/tests/unit/coursier_test.bzl index ea366e08d..403e0a85f 100644 --- a/tests/unit/coursier_test.bzl +++ b/tests/unit/coursier_test.bzl @@ -1,7 +1,11 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") load("//:coursier.bzl", + "add_netrc_entries_from_mirror_urls", + "extract_netrc_from_auth_url", + "get_netrc_lines_from_entries", infer = "infer_artifact_path_from_primary_and_repos", "remove_auth_from_url", + "split_url", ) ALL_TESTS = [] @@ -120,6 +124,190 @@ def _remove_auth_noauth_noop_test_impl(ctx): remove_auth_noauth_noop_test = add_test(_remove_auth_noauth_noop_test_impl) +def _split_url_basic_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ("https", ["c1"]), + split_url("https://c1")) + return unittest.end(env) + +split_url_basic_test = add_test(_split_url_basic_test_impl) + +def _split_url_basic_auth_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ("https", ["a:b@c1"]), + split_url("https://a:b@c1")) + asserts.equals( + env, + ("https", ["a@c1"]), + split_url("https://a@c1")) + return unittest.end(env) + +split_url_basic_auth_test = add_test(_split_url_basic_auth_test_impl) + +def _split_url_with_path_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ("https", ["c1", "some", "path"]), + split_url("https://c1/some/path")) + return unittest.end(env) + +split_url_with_path_test = add_test(_split_url_with_path_test_impl) + +def _extract_netrc_from_auth_url_noop_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {}, + extract_netrc_from_auth_url("https://c1")) + asserts.equals( + env, + {}, + extract_netrc_from_auth_url("https://c2/useless@inurl")) + return unittest.end(env) + +extract_netrc_from_auth_url_noop_test = add_test(_extract_netrc_from_auth_url_noop_test_impl) + +def _extract_netrc_from_auth_url_with_auth_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {"machine": "c", "login": "a", "password": "b"}, + extract_netrc_from_auth_url("https://a:b@c")) + asserts.equals( + env, + {"machine": "c", "login": "a", "password": "b"}, + extract_netrc_from_auth_url("https://a:b@c/some/other/stuff@thisplace/for/testing")) + asserts.equals( + env, + {"machine": "c", "login": "a", "password": None}, + extract_netrc_from_auth_url("https://a@c")) + asserts.equals( + env, + {"machine": "c", "login": "a", "password": None}, + extract_netrc_from_auth_url("https://a@c/some/other/stuff@thisplace/for/testing")) + return unittest.end(env) + +extract_netrc_from_auth_url_with_auth_test = add_test(_extract_netrc_from_auth_url_with_auth_test_impl) + +def _extract_netrc_from_auth_url_at_in_password_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {"machine": "c", "login": "a", "password": "p@ssword"}, + extract_netrc_from_auth_url("https://a:p@ssword@c")) + return unittest.end(env) + +extract_netrc_from_auth_url_at_in_password_test = add_test(_extract_netrc_from_auth_url_at_in_password_test_impl) + +def _add_netrc_entries_from_mirror_urls_noop_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {}, + add_netrc_entries_from_mirror_urls({}, ["https://c1", "https://c1/something@there"])) + return unittest.end(env) + +add_netrc_entries_from_mirror_urls_noop_test = add_test(_add_netrc_entries_from_mirror_urls_noop_test_impl) + +def _add_netrc_entries_from_mirror_urls_basic_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {"c1": {"a": "b"}}, + add_netrc_entries_from_mirror_urls({}, ["https://a:b@c1"])) + asserts.equals( + env, + {"c1": {"a": "b"}}, + add_netrc_entries_from_mirror_urls( + {"c1": {"a": "b"}}, + ["https://a:b@c1"])) + return unittest.end(env) + +add_netrc_entries_from_mirror_urls_basic_test = add_test(_add_netrc_entries_from_mirror_urls_basic_test_impl) + +def _add_netrc_entries_from_mirror_urls_multi_login_ignored_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {"c1": {"a": "b"}}, + add_netrc_entries_from_mirror_urls({}, ["https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"])) + asserts.equals( + env, + {"c1": {"a": "b"}}, + add_netrc_entries_from_mirror_urls( + {"c1": {"a": "b"}}, + ["https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"])) + return unittest.end(env) + +add_netrc_entries_from_mirror_urls_multi_login_ignored_test = add_test(_add_netrc_entries_from_mirror_urls_multi_login_ignored_test_impl) + +def _add_netrc_entries_from_mirror_urls_multi_case_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + {"foo": {"bar": "baz"}, + "c1": {"a1": "b1"}, + "c2": {"a2": "b2"}}, + add_netrc_entries_from_mirror_urls( + {"foo": {"bar": "baz"}}, + ["https://a1:b1@c1", "https://a2:b2@c2", "https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"])) + return unittest.end(env) + +add_netrc_entries_from_mirror_urls_multi_case_test = add_test(_add_netrc_entries_from_mirror_urls_multi_case_test_impl) + +def _get_netrc_lines_from_entries_noop_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + [], + get_netrc_lines_from_entries({})) + return unittest.end(env) + +get_netrc_lines_from_entries_noop_test = add_test(_get_netrc_lines_from_entries_noop_test_impl) + +def _get_netrc_lines_from_entries_basic_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ["machine c", "login a", "password b"], + get_netrc_lines_from_entries({ + "c": {"a": "b"} + })) + return unittest.end(env) + +get_netrc_lines_from_entries_basic_test = add_test(_get_netrc_lines_from_entries_basic_test_impl) + +def _get_netrc_lines_from_entries_no_pass_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ["machine c", "login a"], + get_netrc_lines_from_entries({ + "c": {"a": ""} + })) + return unittest.end(env) + +get_netrc_lines_from_entries_no_pass_test = add_test(_get_netrc_lines_from_entries_no_pass_test_impl) + +def _get_netrc_lines_from_entries_multi_test_impl(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + ["machine c", "login a", "password b", + "machine c2", "login a2", "password p@ssword"], + get_netrc_lines_from_entries({ + "c": {"a": "b"}, + "c2": {"a2": "p@ssword"} + })) + return unittest.end(env) + +get_netrc_lines_from_entries_multi_test = add_test(_get_netrc_lines_from_entries_multi_test_impl) + def coursier_test_suite(): unittest.suite( "coursier_tests",