Merge pull request #766 from ggalmazor/migrate_some_prefs_to_the_stor…

…age_directory Form metadata files and smart append
getodk · Aug 23, 2019 · cd1b966 · cd1b966
2 parents 832c2b8 + 8534059
commit cd1b966
Show file tree

Hide file tree

Showing 58 changed files with 1,613 additions and 325 deletions.
diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml
@@ -65,11 +65,6 @@
             <property name="id" value="RightCurlySame"/>
             <property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/>
         </module>
-        <module name="RightCurly">
-            <property name="id" value="RightCurlyAlone"/>
-            <property name="option" value="alone"/>
-            <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
-        </module>
         <module name="OneStatementPerLine"/>
         <module name="MultipleVariableDeclarations"/>
         <!-- <module name="ArrayTypeStyle"/> -->

diff --git a/src/org/opendatakit/briefcase/export/CsvLine.java b/src/org/opendatakit/briefcase/export/CsvLine.java
@@ -12,14 +12,20 @@
  * - Value equlity rules
  */
 class CsvLine {
+  private final String instanceId;
   private final OffsetDateTime submissionDate;
   private final String line;
 
-  CsvLine(OffsetDateTime submissionDate, String line) {
+  CsvLine(String instanceId, OffsetDateTime submissionDate, String line) {
+    this.instanceId = instanceId;
     this.submissionDate = submissionDate;
     this.line = line;
   }
 
+  public String getInstanceId() {
+    return instanceId;
+  }
+
   public OffsetDateTime getSubmissionDate() {
     return submissionDate;
   }
@@ -30,7 +36,7 @@ public String getLine() {
 
   @Override
   public String toString() {
-    return "csv[" + submissionDate + "] " + line + "]";
+    return "csv[" + instanceId + " - " + submissionDate + "] " + line + "]";
   }
 
   @Override

diff --git a/src/org/opendatakit/briefcase/export/CsvLines.java b/src/org/opendatakit/briefcase/export/CsvLines.java
@@ -1,11 +1,13 @@
 package org.opendatakit.briefcase.export;
 
+import static java.util.Comparator.comparing;
+
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.opendatakit.briefcase.reused.BriefcaseException;
@@ -23,25 +25,29 @@
 class CsvLines {
   private final String modelFqn;
   private final List<CsvLine> lines;
+  private final Optional<CsvLine> lastLine;
 
-  CsvLines(String modelFqn, List<CsvLine> lines) {
+  CsvLines(String modelFqn, List<CsvLine> lines, Optional<CsvLine> lastLine) {
     this.modelFqn = modelFqn;
     this.lines = lines;
+    this.lastLine = lastLine;
   }
 
   /**
    * Factory of an empty instance to be used when reducing a stream of instances.
    */
   static CsvLines empty() {
-    return new CsvLines(null, new ArrayList<>());
+    return new CsvLines(null, new ArrayList<>(), Optional.empty());
   }
 
-  public static CsvLines of(String modelFqn, OffsetDateTime submissionDate, String line) {
-    return new CsvLines(modelFqn, Collections.singletonList(new CsvLine(submissionDate, line)));
+  public static CsvLines of(String modelFqn, String instanceId, OffsetDateTime submissionDate, String line) {
+    CsvLine csvLine = new CsvLine(instanceId, submissionDate, line);
+    return new CsvLines(modelFqn, Collections.singletonList(csvLine), Optional.of(csvLine));
   }
 
-  public static CsvLines of(String modelFqn, OffsetDateTime submissionDate, List<String> lines) {
-    return new CsvLines(modelFqn, lines.stream().map(line -> new CsvLine(submissionDate, line)).collect(Collectors.toList()));
+  public static CsvLines of(String modelFqn, String instanceId, OffsetDateTime submissionDate, List<String> lines) {
+    List<CsvLine> csvLines = lines.stream().map(line -> new CsvLine(instanceId, submissionDate, line)).collect(Collectors.toList());
+    return new CsvLines(modelFqn, csvLines, csvLines.isEmpty() ? Optional.empty() : Optional.of(csvLines.get(csvLines.size() - 1)));
   }
 
   /**
@@ -57,10 +63,14 @@ static CsvLines merge(CsvLines left, CsvLines right) {
     // modelFqns must match or be null
     if (left.modelFqn != null && right.modelFqn != null && !left.modelFqn.equals(right.modelFqn))
       throw new BriefcaseException("FQDN don't match");
-    List<CsvLine> lines = new ArrayList<>();
-    lines.addAll(left.lines);
-    lines.addAll(right.lines);
-    return new CsvLines(coalesce(left.modelFqn, right.modelFqn), lines);
+    List<CsvLine> csvLines = new ArrayList<>();
+    csvLines.addAll(left.lines);
+    csvLines.addAll(right.lines);
+    List<CsvLine> lastCsvLines = new ArrayList<>();
+    left.lastLine.ifPresent(lastCsvLines::add);
+    right.lastLine.ifPresent(lastCsvLines::add);
+    Optional<CsvLine> lastLine = lastCsvLines.stream().max(comparing(CsvLine::getSubmissionDate));
+    return new CsvLines(coalesce(left.modelFqn, right.modelFqn), csvLines, lastLine);
   }
 
   /**
@@ -74,11 +84,15 @@ String getModelFqn() {
     return modelFqn;
   }
 
+  public Optional<CsvLine> getLastLine() {
+    return lastLine;
+  }
+
   /**
    * Return the sorted stream of lines this instance holds
    */
   Stream<String> sorted() {
-    return lines.stream().sorted(Comparator.comparing(CsvLine::getSubmissionDate)).map(CsvLine::getLine);
+    return lines.stream().sorted(comparing(CsvLine::getSubmissionDate)).map(CsvLine::getLine);
   }
 
   /**

diff --git a/src/org/opendatakit/briefcase/export/CsvSubmissionMappers.java b/src/org/opendatakit/briefcase/export/CsvSubmissionMappers.java
@@ -65,6 +65,7 @@ static CsvSubmissionMapper main(FormDefinition formDefinition, ExportConfigurati
         cols.add(submission.getValidationStatus().asCsvValue());
       return CsvLines.of(
           formDefinition.getModel().fqn(),
+          submission.getInstanceId(formDefinition.hasRepeatableFields()),
           submission.getSubmissionDate().orElse(MIN_SUBMISSION_DATE),
           cols.stream().collect(joining(","))
       );
@@ -78,6 +79,7 @@ static CsvSubmissionMapper main(FormDefinition formDefinition, ExportConfigurati
   static CsvSubmissionMapper repeat(FormDefinition formDefinition, Model groupModel, ExportConfiguration configuration) {
     return submission -> CsvLines.of(
         groupModel.fqn(),
+        submission.getInstanceId(),
         submission.getSubmissionDate().orElse(MIN_SUBMISSION_DATE),
         submission.getElements(groupModel.fqn()).stream().map(element -> {
           List<String> cols = new ArrayList<>();

diff --git a/src/org/opendatakit/briefcase/export/ExportConfiguration.java b/src/org/opendatakit/briefcase/export/ExportConfiguration.java
@@ -67,6 +67,7 @@ public class ExportConfiguration {
   private static final String SPLIT_SELECT_MULTIPLES_OVERRIDE = "splitSelectMultiplesOverride";
   private static final String INCLUDE_GEOJSON_EXPORT = "includeGeoJsonExport";
   private static final String REMOVE_GROUP_NAMES = "removeGroupNames";
+  private static final String SMART_APPEND = "smartAppend";
   private final Optional<String> exportFileName;
   private final Optional<Path> exportDir;
   private final Optional<Path> pemFile;
@@ -77,8 +78,9 @@ public class ExportConfiguration {
   private final OverridableBoolean splitSelectMultiples;
   private final OverridableBoolean includeGeoJsonExport;
   private final OverridableBoolean removeGroupNames;
+  private final OverridableBoolean smartAppend;
 
-  private ExportConfiguration(Optional<String> exportFileName, Optional<Path> exportDir, Optional<Path> pemFile, DateRange dateRange, OverridableBoolean pullBefore, OverridableBoolean overwriteFiles, OverridableBoolean exportMedia, OverridableBoolean splitSelectMultiples, OverridableBoolean includeGeoJsonExport, OverridableBoolean removeGroupNames) {
+  private ExportConfiguration(Optional<String> exportFileName, Optional<Path> exportDir, Optional<Path> pemFile, DateRange dateRange, OverridableBoolean pullBefore, OverridableBoolean overwriteFiles, OverridableBoolean exportMedia, OverridableBoolean splitSelectMultiples, OverridableBoolean includeGeoJsonExport, OverridableBoolean removeGroupNames, OverridableBoolean smartAppend) {
     this.exportFileName = exportFileName;
     this.exportDir = exportDir;
     this.pemFile = pemFile;
@@ -89,6 +91,7 @@ private ExportConfiguration(Optional<String> exportFileName, Optional<Path> expo
     this.splitSelectMultiples = splitSelectMultiples;
     this.includeGeoJsonExport = includeGeoJsonExport;
     this.removeGroupNames = removeGroupNames;
+    this.smartAppend = smartAppend;
   }
 
   public static List<String> keys() {
@@ -106,7 +109,8 @@ public static List<String> keys(String keyPrefix) {
         keyPrefix + EXPORT_MEDIA,
         keyPrefix + SPLIT_SELECT_MULTIPLES,
         keyPrefix + INCLUDE_GEOJSON_EXPORT,
-        keyPrefix + REMOVE_GROUP_NAMES
+        keyPrefix + REMOVE_GROUP_NAMES,
+        keyPrefix + SMART_APPEND
     );
   }
 
@@ -153,6 +157,7 @@ public Map<String, String> asMap(String keyPrefix) {
     map.put(keyPrefix + SPLIT_SELECT_MULTIPLES, splitSelectMultiples.serialize());
     map.put(keyPrefix + INCLUDE_GEOJSON_EXPORT, includeGeoJsonExport.serialize());
     map.put(keyPrefix + REMOVE_GROUP_NAMES, removeGroupNames.serialize());
+    map.put(keyPrefix + SMART_APPEND, smartAppend.serialize());
     return map;
   }
 
@@ -192,6 +197,10 @@ boolean resolveRemoveGroupNames() {
     return removeGroupNames.resolve(false);
   }
 
+  boolean resolveSmartAppend() {
+    return smartAppend.resolve(false);
+  }
+
   public OverridableBoolean getPullBefore() {
     return pullBefore;
   }
@@ -216,6 +225,10 @@ public OverridableBoolean getRemoveGroupNames() {
     return removeGroupNames;
   }
 
+  public OverridableBoolean getSmartAppend() {
+    return smartAppend;
+  }
+
   public void ifExportDirPresent(Consumer<Path> consumer) {
     exportDir.ifPresent(consumer);
   }
@@ -233,7 +246,8 @@ public boolean isEmpty() {
         && exportMedia.isEmpty()
         && splitSelectMultiples.isEmpty()
         && includeGeoJsonExport.isEmpty()
-        && removeGroupNames.isEmpty();
+        && removeGroupNames.isEmpty()
+        && smartAppend.isEmpty();
   }
 
   public boolean isValid() {
@@ -252,6 +266,7 @@ ExportConfiguration fallingBackTo(ExportConfiguration defaultConfiguration) {
         .setSplitSelectMultiples(splitSelectMultiples.fallingBackTo(defaultConfiguration.splitSelectMultiples))
         .setIncludeGeoJsonExport(includeGeoJsonExport.fallingBackTo(defaultConfiguration.includeGeoJsonExport))
         .setRemoveGroupNames(removeGroupNames.fallingBackTo(defaultConfiguration.removeGroupNames))
+        .setSmartAppend(smartAppend.fallingBackTo(defaultConfiguration.smartAppend))
         .build();
   }
 
@@ -294,6 +309,7 @@ public String toString() {
         ", splitSelectMultiples=" + splitSelectMultiples +
         ", includeGeoJsonExport=" + includeGeoJsonExport +
         ", removeGroupNames=" + removeGroupNames +
+        ", smartAppend=" + smartAppend +
         '}';
   }
 
@@ -312,12 +328,13 @@ public boolean equals(Object o) {
         Objects.equals(exportMedia, that.exportMedia) &&
         Objects.equals(splitSelectMultiples, that.splitSelectMultiples) &&
         Objects.equals(includeGeoJsonExport, that.includeGeoJsonExport) &&
-        Objects.equals(removeGroupNames, that.removeGroupNames);
+        Objects.equals(removeGroupNames, that.removeGroupNames) &&
+        Objects.equals(smartAppend, that.smartAppend);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(exportDir, pemFile, dateRange, pullBefore, overwriteFiles, exportMedia, splitSelectMultiples, includeGeoJsonExport, removeGroupNames);
+    return Objects.hash(exportDir, pemFile, dateRange, pullBefore, overwriteFiles, exportMedia, splitSelectMultiples, includeGeoJsonExport, removeGroupNames, smartAppend);
   }
 
   public static class Builder {
@@ -332,6 +349,7 @@ public static class Builder {
     private OverridableBoolean splitSelectMultiples = OverridableBoolean.empty();
     private OverridableBoolean includeGeoJsonExport = OverridableBoolean.empty();
     private OverridableBoolean removeGroupNames = OverridableBoolean.empty();
+    private OverridableBoolean smartAppend = OverridableBoolean.empty();
 
     public static Builder empty() {
       return new Builder();
@@ -354,6 +372,7 @@ public static ExportConfiguration load(BriefcasePreferences prefs, String keyPre
           .setSplitSelectMultiples(readOverridableBoolean(prefs, keyPrefix + SPLIT_SELECT_MULTIPLES, keyPrefix + SPLIT_SELECT_MULTIPLES_OVERRIDE))
           .setIncludeGeoJsonExport(readOverridableBoolean(prefs, keyPrefix + INCLUDE_GEOJSON_EXPORT))
           .setRemoveGroupNames(readOverridableBoolean(prefs, keyPrefix + REMOVE_GROUP_NAMES))
+          .setSmartAppend(readOverridableBoolean(prefs, keyPrefix + SMART_APPEND))
           .build();
     }
 
@@ -378,7 +397,8 @@ public ExportConfiguration build() {
           exportMedia,
           splitSelectMultiples,
           includeGeoJsonExport,
-          removeGroupNames
+          removeGroupNames,
+          smartAppend
       );
     }
 
@@ -529,6 +549,16 @@ public Builder setRemoveGroupNames(boolean value) {
       return this;
     }
 
+    public Builder setSmartAppend(OverridableBoolean smartAppend) {
+      this.smartAppend = smartAppend;
+      return this;
+    }
+
+    public Builder setSmartAppend(boolean value) {
+      this.smartAppend = smartAppend.set(value);
+      return this;
+    }
+
     public Builder overridePullBefore(TriStateBoolean overrideValue) {
       pullBefore = pullBefore.overrideWith(overrideValue);
       return this;
@@ -558,5 +588,10 @@ public Builder overrideRemoveGroupNames(TriStateBoolean overrideValue) {
       removeGroupNames = removeGroupNames.overrideWith(overrideValue);
       return this;
     }
+
+    public Builder overrideSmartAppend(TriStateBoolean overrideValue) {
+      smartAppend = smartAppend.overrideWith(overrideValue);
+      return this;
+    }
   }
 }
diff --git a/src/org/opendatakit/briefcase/export/ExportEvent.java b/src/org/opendatakit/briefcase/export/ExportEvent.java
@@ -28,7 +28,7 @@ private ExportEvent(String formId, String statusLine, boolean success) {
   }
 
   public static ExportEvent progress(FormDefinition form, double percentage) {
-    int base100Percentage = new Double(percentage * 100).intValue();
+    int base100Percentage = Double.valueOf(percentage * 100).intValue();
     String statusLine = String.format("Exported %s%% of the submissions", base100Percentage);
     return new ExportEvent(form.getFormId(), statusLine, false);
   }

diff --git a/src/org/opendatakit/briefcase/export/ExportToCsv.java b/src/org/opendatakit/briefcase/export/ExportToCsv.java
@@ -25,18 +25,23 @@
 import static org.opendatakit.briefcase.export.ExportOutcome.ALL_SKIPPED;
 import static org.opendatakit.briefcase.export.ExportOutcome.SOME_SKIPPED;
 import static org.opendatakit.briefcase.export.SubmissionParser.getListOfSubmissionFiles;
+import static org.opendatakit.briefcase.model.form.FormMetadataCommands.updateLastExportedSubmission;
 import static org.opendatakit.briefcase.reused.UncheckedFiles.copy;
 import static org.opendatakit.briefcase.reused.UncheckedFiles.createDirectories;
 import static org.opendatakit.briefcase.reused.UncheckedFiles.deleteRecursive;
 import static org.opendatakit.briefcase.reused.UncheckedFiles.exists;
 import static org.opendatakit.briefcase.reused.UncheckedFiles.write;
 
 import java.nio.file.Path;
+import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.bushe.swing.event.EventBus;
+import org.opendatakit.briefcase.model.FormStatus;
+import org.opendatakit.briefcase.model.form.FormMetadata;
+import org.opendatakit.briefcase.model.form.FormMetadataPort;
 import org.opendatakit.briefcase.ui.reused.Analytics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -47,15 +52,15 @@ public class ExportToCsv {
   /**
    * @see #export(FormDefinition, ExportConfiguration, Optional)
    */
-  public static ExportOutcome export(FormDefinition formDef, ExportConfiguration configuration) {
-    return export(formDef, configuration, Optional.empty());
+  public static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration) {
+    return export(formMetadataPort, formMetadata, formStatus, formDef, briefcaseDir, configuration, Optional.empty());
   }
 
   /**
    * @see #export(FormDefinition, ExportConfiguration, Optional)
    */
-  public static ExportOutcome export(FormDefinition formDef, ExportConfiguration configuration, Analytics analytics) {
-    return export(formDef, configuration, Optional.of(analytics));
+  public static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Analytics analytics) {
+    return export(formMetadataPort, formMetadata, formStatus, formDef, briefcaseDir, configuration, Optional.of(analytics));
   }
 
   /**
@@ -68,7 +73,7 @@ public static ExportOutcome export(FormDefinition formDef, ExportConfiguration c
    * @return an {@link ExportOutcome} with the export operation's outcome
    * @see ExportConfiguration
    */
-  private static ExportOutcome export(FormDefinition formDef, ExportConfiguration configuration, Optional<Analytics> analytics) {
+  private static ExportOutcome export(FormMetadataPort formMetadataPort, FormMetadata formMetadata, FormStatus formStatus, FormDefinition formDef, Path briefcaseDir, ExportConfiguration configuration, Optional<Analytics> analytics) {
     // Create an export tracker object with the total number of submissions we have to export
     ExportProcessTracker exportTracker = new ExportProcessTracker(formDef);
     exportTracker.start();
@@ -79,7 +84,7 @@ private static ExportOutcome export(FormDefinition formDef, ExportConfiguration
             analytics.ifPresent(ga -> ga.event("Export", "Export", "invalid submission", null))
         );
 
-    List<Path> submissionFiles = getListOfSubmissionFiles(formDef, configuration.getDateRange(), onParsingError);
+    List<Path> submissionFiles = getListOfSubmissionFiles(formMetadata, formDef, configuration.getDateRange(), configuration.resolveSmartAppend(), onParsingError);
     exportTracker.trackTotal(submissionFiles.size());
 
     createDirectories(configuration.getExportDir());
@@ -117,6 +122,13 @@ private static ExportOutcome export(FormDefinition formDef, ExportConfiguration
 
     exportTracker.end();
 
+    Optional.ofNullable(csvLinesPerModel.get(formDef.getModel().fqn()))
+        .orElse(CsvLines.empty())
+        .getLastLine()
+        .ifPresent(line -> {
+          formMetadataPort.execute(updateLastExportedSubmission(formMetadata.getKey(), line.getInstanceId(), line.getSubmissionDate(), OffsetDateTime.now(), formStatus.getFormDir(briefcaseDir)));
+        });
+
     ExportOutcome exportOutcome = exportTracker.computeOutcome();
     if (exportOutcome == ALL_EXPORTED)
       EventBus.publish(ExportEvent.successForm(formDef, (int) exportTracker.total));