Last commit to sync with the old repo. contains:

- use reference id for exporting data - Change crypt4gh and clearinghouse settings to use fega-norway repo - rename vars ro increase readability map the datasets with reference to datasets stable id - Handle the cases where the visas are a mix of Fega and non-fega visas - add tests for dataset reference - add tests for visa token
neicnordic · Jan 8, 2025 · 0a3e8a5 · 0a3e8a5
1 parent 2747e97
commit 0a3e8a5
Show file tree

Hide file tree

Showing 12 changed files with 296 additions and 40 deletions.
diff --git a/sda-doa/docker-compose-posix-outbox.yml b/sda-doa/docker-compose-posix-outbox.yml
@@ -10,8 +10,8 @@ services:
     ports:
       - 5432:5432
     environment:
-      - DB_LEGA_IN_PASSWORD=password
-      - DB_LEGA_OUT_PASSWORD=password
+      - LEGA_IN_PASSWORD=password
+      - LEGA_OUT_PASSWORD=password
       - POSTGRES_SERVER_CERT=/etc/ega/pg.cert
       - POSTGRES_SERVER_KEY=/etc/ega/pg.key
       - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert

diff --git a/sda-doa/docker-compose-s3-outbox.yml b/sda-doa/docker-compose-s3-outbox.yml
@@ -7,8 +7,8 @@ services:
     ports:
       - 5432:5432
     environment:
-      - DB_LEGA_IN_PASSWORD=password
-      - DB_LEGA_OUT_PASSWORD=password
+      - LEGA_IN_PASSWORD=password
+      - LEGA_OUT_PASSWORD=password
       - POSTGRES_SERVER_CERT=/etc/ega/pg.cert
       - POSTGRES_SERVER_KEY=/etc/ega/pg.key
       - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert

diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml
@@ -18,6 +18,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.11.0</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-jpa</artifactId>
@@ -109,9 +114,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>no.uio.ifi</groupId>
+            <groupId>no.elixir</groupId>
             <artifactId>crypt4gh</artifactId>
-            <version>2.5.0</version>
+            <version>3.0.5</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -122,7 +127,7 @@
         <dependency>
             <groupId>no.elixir</groupId>
             <artifactId>clearinghouse</artifactId>
-            <version>2.0.1</version>
+            <version>2.0.6</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -139,14 +144,8 @@
 
     <repositories>
         <repository>
-            <id>github-clearinghouse</id>
-            <name>elixir-oslo-clearinghouse</name>
-            <url>https://maven.pkg.github.com/elixir-oslo/clearinghouse</url>
-        </repository>
-        <repository>
-            <id>github-crypt4gh</id>
-            <name>uio-bmi-Crypt4GH</name>
-            <url>https://maven.pkg.github.com/uio-bmi/crypt4gh</url>
+            <id>github-fega-norway</id>
+            <url>https://maven.pkg.github.com/ELIXIR-NO/FEGA-Norway</url>
         </repository>
     </repositories>
 

diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/model/Dataset.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/model/Dataset.java
@@ -0,0 +1,28 @@
+package no.uio.ifi.localega.doa.model;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import org.hibernate.annotations.CacheConcurrencyStrategy;
+import org.hibernate.annotations.Immutable;
+
+@org.hibernate.annotations.Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL)
+@Entity
+@Immutable
+@Getter
+@Setter
+@ToString
+@RequiredArgsConstructor
+@Table(name = "datasets", schema = "sda")
+public class Dataset {
+    @Id
+    private Long id;
+
+    @Column(name = "stable_id", unique = true)
+    private String stableId;
+}
diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java
@@ -0,0 +1,28 @@
+package no.uio.ifi.localega.doa.model;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+import lombok.*;
+import org.hibernate.annotations.CacheConcurrencyStrategy;
+import org.hibernate.annotations.Immutable;
+
+@org.hibernate.annotations.Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL)
+@Entity
+@Immutable
+@Getter
+@Setter
+@ToString
+@RequiredArgsConstructor
+@Table(name = "dataset_references", schema = "sda")
+public class DatasetReferences {
+    @Id
+    private Integer id;
+
+    @Column(name = "dataset_id", nullable = false)
+    private Integer datasetId;
+
+    @Column(name = "reference_id", nullable = false)
+    private String referenceId;
+}
diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java
@@ -73,11 +73,30 @@ public void listen(String message) {
 
             String user = claims.get(Claims.SUBJECT).getAsString();
             log.info("Export request received from user {}: {}", user, exportRequest);
-            Collection<String> datasetIds = aaiService.getDatasetIds(exportRequest.getJwtToken());
-            if (StringUtils.isNotEmpty(exportRequest.getDatasetId())) {
-                exportDataset(user, datasetIds, exportRequest.getDatasetId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate());
+            Collection<String> approvedDatasetIds = aaiService.getDatasetIds(exportRequest.getJwtToken());
+            String requestedDatasetId = exportRequest.getDatasetId();
+            if (StringUtils.isNotEmpty(requestedDatasetId)) {
+                if (metadataService.findByReferenceId(requestedDatasetId) != null) {
+                    Integer datasetsDbTableId = metadataService.findByReferenceId(requestedDatasetId).getDatasetId();
+                    String stableDatasetId = metadataService.getDataset(datasetsDbTableId).getStableId();
+                    log.info("Reference id {} mapped to dataset id {}", requestedDatasetId, stableDatasetId);
+                    requestedDatasetId = stableDatasetId; // use stable dataset id instead of reference to complete the export as normal
+                    Collection<String> approvedMappedDatasetIds = approvedDatasetIds.stream()
+                            .map(x -> {
+                                var reference = metadataService.findByReferenceId(x);
+                                if (reference != null && reference.getDatasetId() != null) {
+                                    var dataset = metadataService.getDataset(reference.getDatasetId());
+                                    return dataset != null ? dataset.getStableId() : x;
+                                } else {
+                                    return x;
+                                }
+                            })
+                            .toList();
+                    approvedDatasetIds = approvedMappedDatasetIds;
+                }
+                exportDataset(user, approvedDatasetIds, requestedDatasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate());
             } else if (StringUtils.isNotEmpty(exportRequest.getFileId())) {
-                exportFile(user, datasetIds, exportRequest.getFileId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate());
+                exportFile(user, approvedDatasetIds, exportRequest.getFileId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate());
             } else {
                 throw new RuntimeException("Either Dataset ID or File ID should be specified");
             }

diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java
@@ -0,0 +1,10 @@
+package no.uio.ifi.localega.doa.repositories;
+
+import no.uio.ifi.localega.doa.model.DatasetReferences;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface DatasetReferencesRepository extends JpaRepository<DatasetReferences, Integer> {
+    DatasetReferences findByReferenceId(String referenceId);
+}
diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java
@@ -0,0 +1,9 @@
+package no.uio.ifi.localega.doa.repositories;
+
+import no.uio.ifi.localega.doa.model.Dataset;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface DatasetsRepository extends JpaRepository<Dataset, Integer> {
+}
diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java
@@ -2,11 +2,11 @@
 
 import lombok.extern.slf4j.Slf4j;
 import no.uio.ifi.localega.doa.dto.File;
+import no.uio.ifi.localega.doa.model.Dataset;
 import no.uio.ifi.localega.doa.model.DatasetEventLog;
+import no.uio.ifi.localega.doa.model.DatasetReferences;
 import no.uio.ifi.localega.doa.model.LEGADataset;
-import no.uio.ifi.localega.doa.repositories.DatasetEventLogRepository;
-import no.uio.ifi.localega.doa.repositories.DatasetRepository;
-import no.uio.ifi.localega.doa.repositories.FileRepository;
+import no.uio.ifi.localega.doa.repositories.*;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -32,6 +32,12 @@ public class MetadataService {
     @Autowired
     private DatasetEventLogRepository datasetEventLogRepository;
 
+    @Autowired
+    private DatasetReferencesRepository datasetReferencesRepository;
+
+    @Autowired
+    private DatasetsRepository datasetsRepository;
+
     /**
      * Returns collection of dataset IDs present in the databse.
      *
@@ -88,4 +94,13 @@ public DatasetEventLog findLatestByDatasetId(String datasetId) {
         return optionalDatasetEventLog.orElse(null);
     }
 
+    public DatasetReferences findByReferenceId(String referenceId) {
+        Optional<DatasetReferences> optionalDatasetReferences = Optional.ofNullable(datasetReferencesRepository.findByReferenceId(referenceId));
+        return optionalDatasetReferences.orElse(null);
+    }
+
+    public Dataset getDataset(Integer id) {
+        return datasetsRepository.findById(id).orElse(null);
+    }
+
 }
diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/StreamingService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/StreamingService.java
@@ -4,13 +4,13 @@
 import io.minio.MinioClient;
 import io.minio.errors.*;
 import lombok.extern.slf4j.Slf4j;
-import no.uio.ifi.crypt4gh.pojo.header.DataEditList;
-import no.uio.ifi.crypt4gh.pojo.header.Header;
-import no.uio.ifi.crypt4gh.pojo.header.HeaderPacket;
-import no.uio.ifi.crypt4gh.pojo.header.X25519ChaCha20IETFPoly1305HeaderPacket;
-import no.uio.ifi.crypt4gh.stream.Crypt4GHInputStream;
-import no.uio.ifi.crypt4gh.util.Crypt4GHUtils;
-import no.uio.ifi.crypt4gh.util.KeyUtils;
+import no.elixir.crypt4gh.pojo.header.DataEditList;
+import no.elixir.crypt4gh.pojo.header.Header;
+import no.elixir.crypt4gh.pojo.header.HeaderPacket;
+import no.elixir.crypt4gh.pojo.header.X25519ChaCha20IETFPoly1305HeaderPacket;
+import no.elixir.crypt4gh.stream.Crypt4GHInputStream;
+import no.elixir.crypt4gh.util.Crypt4GHUtils;
+import no.elixir.crypt4gh.util.KeyUtils;
 import no.uio.ifi.localega.doa.dto.DestinationFormat;
 import no.uio.ifi.localega.doa.model.LEGADataset;
 import no.uio.ifi.localega.doa.model.LEGAFile;