Add new argument to allow setting the google requester pays project

src/main/java/picard/cmdline/CommandLineProgram.java

@@ -53,20 +53,20 @@
 import org.broadinstitute.barclay.argparser.SpecialArgumentsCollection;
 import picard.cmdline.argumentcollections.OptionalReferenceArgumentCollection;
 import picard.cmdline.argumentcollections.ReferenceArgumentCollection;
+import picard.cmdline.argumentcollections.RequesterPaysArgumentCollection;
 import picard.cmdline.argumentcollections.RequiredReferenceArgumentCollection;
-import picard.nio.PathProvider;
+import picard.nio.GoogleStorageUtils;
+import picard.nio.HttpNioUtils;
 import picard.nio.PicardHtsPath;
 import picard.util.RExecutor;
 
 import java.io.File;
 import java.net.InetAddress;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * Abstract class to facilitate writing command-line programs.
@@ -85,8 +85,8 @@
  *
  */
 public abstract class CommandLineProgram {
-    private static String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser";
-    private static String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine";
+    private static final String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser";
+    private static final String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine";
     private static Boolean useLegacyParser;
     public static String SYNTAX_TRANSITION_URL =
             "https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition)";
@@ -132,6 +132,9 @@ public abstract class CommandLineProgram {
     // after argument parsing using the value established by the user in the referenceSequence argument collection.
     protected File REFERENCE_SEQUENCE = Defaults.REFERENCE_FASTA;
 
+    @ArgumentCollection
+    public RequesterPaysArgumentCollection requesterPays = new RequesterPaysArgumentCollection();
+
     @ArgumentCollection(doc="Special Arguments that have meaning to the argument parsing system.  " +
                 "It is unlikely these will ever need to be accessed by the command line program")
     public Object specialArgumentsCollection = useLegacyParser() ?
@@ -182,7 +185,7 @@ public void instanceMainWithExit(final String[] argv) {
     }
 
     public int instanceMain(final String[] argv) {
-        String actualArgs[] = argv;
+        String[] actualArgs = argv;
 
         if (System.getProperty(PROPERTY_CONVERT_LEGACY_COMMAND_LINE, "false").equals("true")) {
             actualArgs = CommandLineSyntaxTranslater.convertPicardStyleToPosixStyle(argv);
@@ -249,16 +252,15 @@ public int instanceMain(final String[] argv) {
         // default reader factory.  At least until https://github.com/samtools/htsjdk/issues/1666 is resolved
         SamReaderFactory.setDefaultValidationStringency(VALIDATION_STRINGENCY);
 
+        // Configure the various filesystem providers
+        GoogleStorageUtils.initialize(requesterPays.getProjectForRequesterPays());
+        HttpNioUtils.initialize();
+
         if (!QUIET) {
             System.err.println("[" + new Date() + "] " + commandLine);
 
-            // Output a one liner about who/where and what software/os we're running on
+            // Output a one-liner about who/where and what software/os we're running on
             try {
-                final String pathProvidersMessage =
-                        Arrays.stream(PathProvider.values())
-                                .map(provider -> String.format("Provider %s is%s available;", provider.name(), provider.isAvailable ? "" : " not"))
-                                .collect(Collectors.joining(" "));
-
                 final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory &&
                         ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater());
                 final boolean usingIntelInflater = (BlockGunzipper.getDefaultInflaterFactory() instanceof IntelInflaterFactory &&
@@ -269,7 +271,7 @@ public int instanceMain(final String[] argv) {
                     System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"),
                     System.getProperty("java.vm.name"), System.getProperty("java.runtime.version"),
                     usingIntelDeflater ? "Intel" : "Jdk", usingIntelInflater ? "Intel" : "Jdk",
-                        pathProvidersMessage,
+                    requesterPays.getDescription(),
                     getCommandLineParser().getVersion());
                 System.err.println(msg);
             }
@@ -409,7 +411,7 @@ public CommandLineParser getCommandLineParser() {
             commandLineParser = useLegacyParser() ?
                     new LegacyCommandLineArgumentParser(this) :
                     new CommandLineArgumentParser(this,
-                        Collections.EMPTY_LIST,
+                        Collections.emptyList(),
                         Collections.singleton(CommandLineParserOptions.APPEND_TO_COLLECTIONS));
         }
         return commandLineParser;
@@ -427,9 +429,7 @@ public CommandLineParser getCommandLineParser() {
     public static boolean useLegacyParser() {
         if (useLegacyParser == null) {
             final String legacyPropertyValue = System.getProperty(PROPERTY_USE_LEGACY_PARSER);
-            useLegacyParser = legacyPropertyValue == null ?
-                    false :
-                    Boolean.parseBoolean(legacyPropertyValue);
+            useLegacyParser = Boolean.parseBoolean(legacyPropertyValue);
         }
         return useLegacyParser;
     }
@@ -475,7 +475,7 @@ public List<Header> getDefaultHeaders() {
     public static String getStandardUsagePreamble(final Class<?> mainClass) {
         return "USAGE: " + mainClass.getSimpleName() +" [options]\n\n" +
                 (hasWebDocumentation(mainClass) ?
-                        "Documentation: http://broadinstitute.github.io/picard/command-line-overview.html" +
+                        "Documentation: https://broadinstitute.github.io/picard/command-line-overview.html" +
                                 mainClass.getSimpleName() + "\n\n" :
                         "");
     }
@@ -499,7 +499,7 @@ public static boolean hasWebDocumentation(final Class<?> clazz){
      * @return the link to a FAQ
      */
     public static String getFaqLink() {
-        return "To get help, see http://broadinstitute.github.io/picard/index.html#GettingHelp";
+        return "To get help, see https://broadinstitute.github.io/picard/index.html#GettingHelp";
     }
 
     /**

src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java

@@ -0,0 +1,42 @@
+package picard.cmdline.argumentcollections;
+
+import com.google.common.base.Strings;
+import org.broadinstitute.barclay.argparser.Argument;
+
+/**
+ * Argument collection to encapsulate special behavior of the REQUESTER_PAYS_PROJECT argument
+ */
+public class RequesterPaysArgumentCollection {
+    /** The System property which acts as a default for REQUESTER_PAYS_PROJECT */
+    public static final String PROPERTY_NAME = "picard.googleProjectForRequesterPays";
+    public static final String REQUESTER_PAYS_PROJECT_FULL_NAME = "REQUESTER_PAYS_PROJECT";
+
+    @Argument(doc="Google project for access to 'requester pays' buckets and objects.  " +
+            "If this is not specified then value of the system property " + PROPERTY_NAME + " acts as the default.",
+            common = true, optional = true,
+    fullName = REQUESTER_PAYS_PROJECT_FULL_NAME)
+    public String requesterPaysProject = null;
+
+    public String getProjectForRequesterPays() {
+        final String value = ! Strings.isNullOrEmpty(requesterPaysProject)
+                ? requesterPaysProject
+                : getSystemProperty();
+        return Strings.isNullOrEmpty(value) ? null : value; // "" -> null
+    }
+
+    private String getSystemProperty() {
+        return System.getProperty(PROPERTY_NAME);
+    }
+
+    public String getDescription(){
+         final String value = getProjectForRequesterPays();
+         if(!Strings.isNullOrEmpty(requesterPaysProject)){
+             return "Requester Pays Project set by argument: " + value;
+         } else if( !Strings.isNullOrEmpty(getSystemProperty())){
+             return "Requester Pays Project set by system property: " + value;
+         } else {
+             return "Requester Pays Project not set.";
+         }
+    }
+
+}

src/main/java/picard/nio/GoogleStorageUtils.java

@@ -36,22 +36,26 @@
 /**
  * This class serves as a connection to google's implementation of nio support for GCS housed files.
  *
- * While the actual code required to connect isn't packaged with Picard (only compiled), the Readme.md file in the
- * github repository describes how it can be used. Additionally, Picard is bundled in GATK4, and its tools exposed via
- * the GATK engine, since the nio library _is_ included the GATK4 jar. NIO enabled tools in picard can connect to
- * GCS when called through GATK4.
- *
  * This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics
  * work. In the future it would make sense to expose these parameters so that they can be controlled via the commandline.
  * However, as the list of Path-enabled tools in picard is small, there seems to be little impetus to do so right now.
  *
  *
  */
-class GoogleStorageUtils {
+public final class GoogleStorageUtils {
+
+    public static final int MAX_REOPENS = 20;
+
+    private GoogleStorageUtils(){}
 
-    public static void initialize() {
+    /**
+     * Set appropriate configuration options for the GCS file system provider.
+     *
+     * @param requesterProject the project to pay with when accessing requester pays buckets
+     */
+    public static void initialize(final String requesterProject) {
         // requester pays support is currently not configured
-        CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(20, null));
+        CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(MAX_REOPENS, requesterProject));
         CloudStorageFileSystemProvider.setStorageOptions(GoogleStorageUtils.setGenerousTimeouts(StorageOptions.newBuilder()).build());
     }
 
@@ -62,7 +66,8 @@ private static CloudStorageConfiguration getCloudStorageConfiguration(int maxReo
                 .maxChannelReopens(maxReopens);
         if (!Strings.isNullOrEmpty(requesterProject)) {
             // enable requester pays and indicate who pays
-            builder = builder.autoDetectRequesterPays(true).userProject(requesterProject);
+            builder.autoDetectRequesterPays(true)
+                    .userProject(requesterProject);
         }
 
         // this causes the gcs filesystem to treat files that end in a / as a directory

src/main/java/picard/nio/HttpNioUtils.java

@@ -0,0 +1,37 @@
+package picard.nio;
+
+import org.broadinstitute.http.nio.HttpFileSystemProvider;
+import org.broadinstitute.http.nio.HttpFileSystemProviderSettings;
+import org.broadinstitute.http.nio.RetryHandler;
+
+import java.time.Duration;
+
+/**
+ * This class provides a way to easily configure the HttpNioProvider
+ *
+ * This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics
+ * work.
+ *
+ */
+public final class HttpNioUtils {
+
+    private HttpNioUtils() {}
+    public static final Duration MAX_TIMEOUT = Duration.ofMillis(120_000);
+    public static final int MAX_RETRIES = 20;
+
+    public static void initialize() {
+        final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings(
+                MAX_RETRIES,
+                RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES,
+                RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS,
+                RetryHandler.DEFALT_RETRYABLE_MESSAGES,
+                e -> false);
+
+        final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings(
+                MAX_TIMEOUT,
+                HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(),
+                retrySettings);
+
+        HttpFileSystemProvider.setSettings(settings);
+    }
+}

src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java

@@ -0,0 +1,70 @@
+package picard.cmdline.argumentcollections;
+
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class RequesterPaysArgumentCollectionTest {
+
+    public static final String P1 = "project1";
+    public static final String P2 = "project2";
+
+    @DataProvider
+    public static Object[][] settings() {
+
+        return new Object[][]{
+                {null, null, null},
+                {"", null, null},
+                {null, "", null},
+                {"", "", null},
+                {P1, null, P1},
+                {null, P2, P2},
+                {P1, P2, P1},
+                {"", P2, P2},
+                {P1, "", P1}
+        };
+    }
+
+    @Test(dataProvider = "settings")
+    public void testCorrectValues(String arg, String sys, String expected){
+        runWithSystemProperty(
+                () -> {
+                    final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection();
+                    rpc.requesterPaysProject = arg;
+                    Assert.assertEquals(rpc.getProjectForRequesterPays(), expected);
+                }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys
+        );
+    }
+
+    @Test(dataProvider = "settings")
+    public void testCorrectDescription(String arg, String sys, String expected){
+        runWithSystemProperty(
+                () -> {
+                    final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection();
+                    rpc.requesterPaysProject = arg;
+                    final String description = rpc.getDescription();
+                    final String value = rpc.getProjectForRequesterPays();
+                    if(expected == null) {Assert.assertEquals(description, "Requester Pays Project not set."); }
+                    else if(expected.equals(P1)) { Assert.assertEquals(description, "Requester Pays Project set by argument: " + value); }
+                    else if(expected.equals(P2)) { Assert.assertEquals(description, "Requester Pays Project set by system property: " + value); }
+                    else { Assert.fail("should have been one of the of the previous"); }
+                }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys);
+    }
+    private static void runWithSystemProperty(Runnable toRun, String name, String value){
+        String previousValue = null;
+        try {
+            if(value != null) {
+                previousValue = System.setProperty(name, value);
+            }
+
+            toRun.run();
+
+        } finally {
+            if(previousValue == null){
+                System.clearProperty(name);
+            } else {
+                System.setProperty(name, previousValue);
+            }
+        }
+    }
+}