Improve error catching of Nahima Export

pom.xml

@@ -136,6 +136,11 @@
 			<artifactId>javase</artifactId>
 			<version>3.4.0</version>
 		</dependency>
+		<dependency>
+			<groupId>com.github.mizosoft.methanol</groupId>
+			<artifactId>methanol</artifactId>
+			<version>1.6.0</version>
+		</dependency>
 		<dependency>
 			<groupId>com.itextpdf</groupId>
 			<artifactId>itextpdf</artifactId>

src/main/java/edu/harvard/mcz/imagecapture/data/NahimaManager.java

@@ -1,18 +1,19 @@
 package edu.harvard.mcz.imagecapture.data;
 
-import edu.harvard.mcz.imagecapture.ImageCaptureApp;
+import com.github.mizosoft.methanol.MediaType;
+import com.github.mizosoft.methanol.MultipartBodyPublisher;
+import com.github.mizosoft.methanol.MutableRequest;
 import edu.harvard.mcz.imagecapture.ImageCaptureProperties;
 import edu.harvard.mcz.imagecapture.entity.ICImage;
 import edu.harvard.mcz.imagecapture.entity.Specimen;
 import edu.harvard.mcz.imagecapture.utility.AbstractRestClient;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.net.URI;
-import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.file.Path;
 import java.util.*;
@@ -23,6 +24,7 @@ public class NahimaManager extends AbstractRestClient {
     private final String url;
     private final String username;
     private final String password;
+    public static final JSONObject defaultPool = new JSONObject("{ \"pool\": { \"_id\": 7 } }");
     private String token;
     private Map<String, Map<String, JSONObject>> resolveCache = new HashMap<String, Map<String, JSONObject>>();
 
@@ -54,44 +56,63 @@ protected void startSessionAndRetrieveToken() throws RuntimeException {
         }
     }
 
-    protected void login() throws IOException, InterruptedException {
-        String queryUrl = this.url + "session/authenticate?token=" + this.token;
+    protected void login() throws IOException, InterruptedException, RuntimeException {
+        String queryUrl = this.url + "session/authenticate?method=easydb&token=" + this.token;
         HashMap<String, String> params = new HashMap<>();
-        params.put("username", this.username);
         params.put("password", this.password);
-
-        this.postRequest(queryUrl, params);
+        params.put("login", this.username);
+
+        // check status
+        String response = this.postRequest(queryUrl, params);
+        JSONObject responseObject = new JSONObject(response);
+        if (responseObject.has("code")) {
+            if (((String) responseObject.get("code")).startsWith("error")) {
+                throw new RuntimeException("Failed to log in. Error code: " + responseObject.get("code"));
+            }
+        }
     }
 
-    public JSONObject[] uploadImagesForSpecimen(Specimen specimen) throws IOException, InterruptedException {
+    public Object[] uploadImagesForSpecimen(Specimen specimen) throws IOException, InterruptedException, RuntimeException {
         // docs:
         // https://docs.easydb.de/en/technical/api/eas/
         // https://docs.easydb.de/en/sysadmin/eas/api/put/
         String baseQueryUrl = this.url + "eas/put?token=" + this.token;
-        ArrayList results = new ArrayList<>();
+        ArrayList<Object> results = new ArrayList<>();
 
         for (ICImage image : specimen.getICImages()) {
             String queryUrl = baseQueryUrl + "&original_filename=" + image.getFilename() + "&instance=image";
+            log.debug("Running image upload to URL " + queryUrl);
 
-            HttpRequest.Builder builder = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofFile(Path.of(ImageCaptureProperties.assemblePathWithBase(image.getPath(), image.getFilename())))).uri(URI.create(url))
-                    .setHeader("User-Agent", "DataShot " + ImageCaptureApp.getAppVersion()) // add request header
-                    .setHeader("Content-Type", "application/x-www-form-urlencoded");
-
-            HttpRequest request = builder.build();
+            String imagePath = ImageCaptureProperties.assemblePathWithBase(image.getPath(), image.getFilename());
 
+            MultipartBodyPublisher multipartBody = MultipartBodyPublisher.newBuilder().filePart("image", Path.of(imagePath), MediaType.IMAGE_ANY).build();
+            MutableRequest request = MutableRequest.POST(queryUrl, multipartBody).header("Content-Disposition", "attachment; filename=\"" + image.getFilename() + "\"");
             HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+            log.debug("Response from uploading image: " + response.body());
+
+            // check for errors
+            if (!response.body().startsWith("[")) {
+                JSONObject responseObject = new JSONObject(response.body());
+                if (responseObject.has("code") && responseObject.getString("code").startsWith("error")) {
+                    throw new RuntimeException("Failed to upload image: Error code: " + responseObject.get("code"));
+                }
+            }
+
             results.add((new JSONArray(response.body())).get(0));
         }
 
-        return (JSONObject[]) results.toArray();
+        return results.toArray();
     }
 
     public JSONObject createObjectInNahima(JSONObject object, String pool) throws IOException, InterruptedException {
         // docs:
         // https://docs.easydb.de/en/technical/api/db/
         String queryURL = this.url + "db/" + pool + "?token=" + token;
         // EasyDB seems to have mixed up PUT & POST, but well, we don't care
-        return new JSONObject(this.putRequest(queryURL, (new JSONArray()).put(object).toString(), null));
+        return new JSONObject(this.putRequest(queryURL, (new JSONArray()).put(object).toString(), new HashMap<>() {{
+            put("Content-Type", "application/json");
+        }}));
     }
 
     public JSONObject searchForString(String searchString, String objectType) throws IOException, InterruptedException {
@@ -102,8 +123,8 @@ public JSONObject searchForString(String searchString, String objectType) throws
      * Search for a string in a Nahima objecttype
      *
      * @param searchString the string to search for. Will be split by " ".
-     * @param objectType the objecttype we search for
-     * @param ignoreCache cache speeds things up, but is annoying when we just created a new object
+     * @param objectType   the objecttype we search for
+     * @param ignoreCache  cache speeds things up, but is annoying when we just created a new object
      * @return Nahima's response
      * @throws IOException
      * @throws InterruptedException
@@ -186,10 +207,18 @@ public JSONObject reduceAssociateForAssociation(JSONObject associate) {
         JSONObject reduced = new JSONObject();
         String[] namesToKeep = {"_objecttype", "_mask", "_global_object_id"};
         for (String name : namesToKeep) {
-            reduced.put(name, associate.get(name));
+            try {
+                if (associate.has(name)) {
+                    reduced.put(name, associate.get(name));
+                }
+            } catch (JSONException e) {
+                // hmmm
+                log.error("Error trying to reduce", e);
+            }
         }
         JSONObject child = new JSONObject();
-        child.put("_id", ((JSONObject) associate.get((String) associate.get("_objecttype"))).get("_id"));
+
+        child.put("_id", associate.has("_id") ? associate.get("_id") : ((JSONObject) associate.get((String) associate.get("_objecttype"))).get("_id"));
         reduced.put((String) associate.get("_objecttype"), child);
         return reduced;
     }

src/main/java/edu/harvard/mcz/imagecapture/jobs/NahimaExportJob.java

@@ -26,11 +26,11 @@ public class NahimaExportJob implements RunnableJob, Runnable {
     public static int STATUS_DATASHOT_FAILED = 1;
     public static int STATUS_RUNNING_FINE = 0;
     public static int STATUS_FINISHED = -1;
+    private final List<ProgressListener> progressListener = new ArrayList<>();
     private Date start = null;
     private int nrOfSpecimenToProcess = 0;
     private int currentIndex = 0;
     private int status = STATUS_RUNNING_FINE;
-    private final List<ProgressListener> progressListener = new ArrayList<>();
     private Exception lastError = null;
 
     @Override
@@ -61,7 +61,7 @@ public void start() {
             currentIndex = currentIndex + 1;
             log.debug("Exporting specimen with id " + specimen.getSpecimenId() + " and barcode " + specimen.getBarcode());
             // upload images
-            JSONObject[] uploadedImages;
+            Object[] uploadedImages;
             try {
                 uploadedImages = manager.uploadImagesForSpecimen(specimen);
             } catch (Exception e) {
@@ -76,18 +76,30 @@ public void start() {
             // add images
             JSONObject imageTarget = (JSONObject) specimenJson.get("entomologie");
             JSONArray mediaAssets = new JSONArray();
-            for (JSONObject image : uploadedImages) {
+            for (Object image : uploadedImages) {
+                JSONObject imageJsonObj = (JSONObject) image;
+                Map<String, Object> reducedMediaAssetMap = new HashMap<>() {{
+                    put("_pool", NahimaManager.defaultPool);
+                    put("_id", JSONObject.NULL);
+                    put("_version", 1);
+                    put("datei", (new JSONArray()).put(new JSONObject(new HashMap<>() {{
+                        put("preferred", true);
+                        put("_id", imageJsonObj.get("_id"));
+                    }})));
+                    put("urspruelicherdateiname", imageJsonObj.get("original_filename"));
+                }};
+                JSONObject reducedImageMediaAsset = new JSONObject(reducedMediaAssetMap);
                 Map<String, Object> publicMediaAssetMap = new HashMap<>() {{
                     put("_objecttype", "mediaasset");
                     put("_mask", "mediaassetminimal");
                     put("_tags", new JSONArray());
-                    put("mediaasset", manager.reduceAssociateForAssociation(image));
+                    put("mediaasset", reducedImageMediaAsset);
                 }};
                 JSONObject publicMediaAsset = new JSONObject(publicMediaAssetMap);
 
                 Map<String, Object> mediaAssetMap = new HashMap<>() {{
-                    put("_id", null);
-                    put("source_reference", null);
+                    put("_id", JSONObject.NULL);
+                    put("source_reference", JSONObject.NULL);
                     put("__idx", 0);
                     put("_version", 1);
                     put("mediassetpublic", publicMediaAsset);
@@ -99,8 +111,11 @@ public void start() {
 
             // create in Nahima
             try {
-                manager.createObjectInNahima(specimenJson, "entomologie");
-            } catch (IOException | InterruptedException e) {
+                JSONObject result = manager.createObjectInNahima(specimenJson, "entomologie");
+                if (result.has("code") && result.getString("code").startsWith("error")) {
+                    throw new RuntimeException("Failed to save specimen in nahima. Error code: " + result.get("code"));
+                }
+            } catch (IOException | InterruptedException | RuntimeException e) {
                 lastError = e;
                 log.error("Failed to create new Nahima specimen", e);
                 this.status = STATUS_NAHIMA_FAILED;

src/main/java/edu/harvard/mcz/imagecapture/serializer/nahima/Specimen2JSONSerializer.java

@@ -40,10 +40,19 @@ public JSONObject serialize2JSON(Object target) {
         result.put("folgerung", toSerialize.getInferences());
         // EasyDB specific fields
         result.put("_version", 1);
+        result.put("_id", JSONObject.NULL);
+        result.put("_pool", NahimaManager.defaultPool);
         // here, it already starts to get complicated, fuu.
         JSONArray reverseNestedDeterminations = new JSONArray();
         for (Determination det : toSerialize.getDeterminations()) {
-            JSONObject reverseNestedDetermination = new JSONObject();
+            Map<String, Object> reverseNestedCollectionMap = new HashMap<>(){{
+                put("_version", 1);
+                put("_pool", NahimaManager.defaultPool);
+                put("_id", JSONObject.NULL);
+                put("typusid", det.getTypeStatus());
+            }};
+
+            JSONObject reverseNestedDetermination = new JSONObject(reverseNestedCollectionMap);
 
             reverseNestedDetermination.put("_nested:bestimmung__kommentarezurbestimmung", new JSONArray(new String[]{toSerialize.getIdentificationRemarks()}));
             reverseNestedDetermination.put("bestimmungsdatumtrans", det.getDateIdentified());
@@ -56,7 +65,6 @@ public JSONObject serialize2JSON(Object target) {
 
             reverseNestedDetermination.put("autortrans", det.getAuthorship()); // TODO: resolve
             reverseNestedDetermination.put("taxonnametrans", toSerialize.getAssociatedTaxon()); // TODO: resolve
-            reverseNestedDetermination.put("typusid", det.getTypeStatus());
             reverseNestedDetermination.put("familietrans", toSerialize.getFamily()); // TODO: resolve
             reverseNestedDetermination.put("genustrans", det.getGenus()); // TODO: resolve
             reverseNestedDetermination.put("subspezifischesarttrans", det.getSubspecificEpithet()); // TODO: resolve
@@ -96,6 +104,10 @@ public JSONObject serialize2JSON(Object target) {
             put("fehlerradius", georef == null ? null : georef.getMaxErrorDistance());
             put("hoehemin", toSerialize.getMinimum_elevation());
             put("hoehemax", toSerialize.getMaximum_elevation());
+            put("__idx", 0);
+            put("_version", 1);
+            put("_id", JSONObject.NULL);
+            put("_pool", NahimaManager.defaultPool);
         }};
         JSONObject reverseNestedCollection = new JSONObject(reverseNestedCollectionMap);
 
@@ -129,7 +141,7 @@ public JSONObject serialize2JSON(Object target) {
         // finally, wrap everything in the pool (might want to do somewhere else)
         JSONObject wrapper = new JSONObject();
         wrapper.put("entomologie", result);
-        wrapper.put("mask", "entomologie_public_unrestricted");
+        wrapper.put("_mask", "entomologie_public_unrestricted");
         wrapper.put("_objecttype", "entomologie");
         wrapper.put("_idx_in_objects", 1);
         return wrapper;
@@ -145,6 +157,7 @@ protected JSONObject dateRangeToNahima(String from, String to) throws ParseExcep
     }
 
     protected JSONObject dateToNahima(String date) throws ParseException {
+        log.debug("Parsing date from \"" + date + "\"");
         Date parsedDate = DateUtils.parseDate(date);
         return dateToNahima(parsedDate);
     }

src/main/java/edu/harvard/mcz/imagecapture/ui/dialog/NahimaExportDialog.java

@@ -25,22 +25,25 @@ public NahimaExportDialog(Frame owner) {
     }
 
     private void initialize() {
-        this.setTitle("Melia Statistics");
+        this.setTitle("Nahima Export");
         this.setContentPane(getJContentPane());
         this.pack();
         job = new NahimaExportJob();
         job.addProgressListener(this);
         Thread runningThread = new Thread(() -> {
+            boolean didRunSuccessfully = true;
             try {
                 job.run();
             } catch (Exception e) {
+                didRunSuccessfully = false;
                 log.error("Error when running Nahima export job", e);
                 getResultsTextField().setText("Error: " + e.getMessage());
                 getCloseButton().setEnabled(true);
             }
             if (job.getLastException() != null) {
+                didRunSuccessfully = false;
                 getResultsTextField().setText("Error: " + job.getLastException().getMessage());
-            } else {
+            } else if (didRunSuccessfully) {
                 getResultsTextField().setText("Finished processing " + String.valueOf(job.getTotalCount()) + " Specimen");
             }
             getCloseButton().setEnabled(true);
@@ -70,6 +73,8 @@ private JTextArea getResultsTextField() {
         if (resultsTextField == null) {
             resultsTextField = new JTextArea();
             resultsTextField.setEnabled(false);
+            resultsTextField.setLineWrap(true);
+            resultsTextField.setRows(5);
             resultsTextField.setText("Please do not close this window until the export is done.");
         }
         return resultsTextField;

src/main/java/edu/harvard/mcz/imagecapture/utility/AbstractRestClient.java

@@ -1,5 +1,6 @@
 package edu.harvard.mcz.imagecapture.utility;
 
+import com.github.mizosoft.methanol.Methanol;
 import edu.harvard.mcz.imagecapture.ImageCaptureApp;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -24,11 +25,10 @@
 public abstract class AbstractRestClient {
     private static final Logger log =
             LoggerFactory.getLogger(AbstractRestClient.class);
-    protected final HttpClient httpClient = HttpClient.newBuilder()
-            .version(HttpClient.Version.HTTP_2)
-            .build();
+    protected final Methanol httpClient;
 
     public AbstractRestClient() {
+        httpClient = Methanol.newBuilder().userAgent("DataShot " + ImageCaptureApp.getAppVersion()).build();
     }
 
     private static String buildFormDataFromMap(Map<String, String> data) {
@@ -44,7 +44,7 @@ private static String buildFormDataFromMap(Map<String, String> data) {
         return builder.toString();
     }
 
-    protected String getRequest(String url) throws Exception {
+    protected String getRequest(String url) throws IOException {
         return getRequest(url, null);
     }
 
@@ -93,6 +93,9 @@ protected String getRequest(String url, Map<String, String> headers) throws IOEx
         }
 
         con.disconnect();
+
+        log.debug("Response of put request: " + response.toString());
+
         return response.toString();
     }
 
@@ -138,6 +141,7 @@ protected String postRequest(String url, String data, Map<String, String> header
         HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
 
         // System.out.println(response.statusCode());
+        log.debug("Response of post request: " + response.body());
 
         return response.body();
     }
@@ -194,6 +198,7 @@ protected String putRequest(String url, String data, Map<String, String> headers
 
         HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
 
+        log.debug("Response of put request: " + response.body());
         // System.out.println(response.statusCode());
         return response.body();
     }
@@ -264,7 +269,7 @@ protected Map<String, Object> fetchValues(String url,
             log.error("Error", e);
         }
 
-        log.debug("obj=" + obj);
+        log.debug("Fetched obj=" + obj);
 
         if (obj != null) {
             if (obj.length() > 0) {