Update to new ODF schema

kamu-data · Aug 22, 2020 · 81ac055 · 81ac055
1 parent f672311
commit 81ac055
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.23.0] - 2020-08-22
+### Changed
+- Updated to latest ODF schema
+
 ## [0.22.0] - 2020-07-15
 ### Added
 - `#26`: Follow redirects when fetching data from URL

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala b/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala
@@ -76,9 +76,12 @@ class AddInteractiveCommand(
             "Compression",
             "What's the compression format?",
             Seq("zip", "gzip")
-          )
+          ) match {
+            case "zip"  => CompressionFormat.Zip
+            case "gzip" => CompressionFormat.Gzip
+          }
 
-          val subPath = if (Seq("zip").contains(compression)) {
+          val subPath = if (compression == CompressionFormat.Zip) {
             inputOptional(
               "Sub-path",
               "If this archive can contain multiple files - specify the path regex to " +

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala b/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala
@@ -30,7 +30,7 @@ class DependencyGraphCommand(
 
     datasets.foreach(
       ds =>
-        ds.datasetDependencies
+        ds.dependencies
           .foreach(d => println(s"${quote(d)} -> ${quote(ds.id)};"))
     )
 

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala
@@ -103,14 +103,14 @@ class SourceFactory(systemClock: Clock) {
       .toVector
 
     val orderBy = kind.order.getOrElse(
-      if (kind.eventTime.isDefined) SourceOrdering.ByEventTime()
-      else SourceOrdering.ByName()
+      if (kind.eventTime.isDefined) SourceOrdering.ByEventTime
+      else SourceOrdering.ByName
     )
 
     val sorted = orderBy match {
-      case SourceOrdering.ByName() =>
+      case SourceOrdering.ByName =>
         globbed.sortBy(_.path.getFileName.toString.toLowerCase())
-      case SourceOrdering.ByEventTime() =>
+      case SourceOrdering.ByEventTime =>
         globbed.sortBy(eventTimeSource.getEventTime)
     }
 

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala
@@ -8,7 +8,7 @@
 
 package dev.kamu.cli.ingest.prep
 
-import dev.kamu.core.manifests.{PrepStep => PrepStepCfg}
+import dev.kamu.core.manifests.{PrepStep => PrepStepCfg, CompressionFormat}
 import org.apache.logging.log4j.LogManager
 
 class PrepStepFactory() {
@@ -19,17 +19,13 @@ class PrepStepFactory() {
   ): PrepStep = {
     config match {
       case dc: PrepStepCfg.Decompress =>
-        dc.format.toLowerCase match {
-          case "gzip" =>
+        dc.format match {
+          case CompressionFormat.Gzip =>
             logger.debug("Extracting gzip")
             new DecompressGZIPStep()
-          case "zip" =>
+          case CompressionFormat.Zip =>
             logger.debug("Extracting zip")
             new DecompressZIPStep(dc)
-          case _ =>
-            throw new NotImplementedError(
-              s"Unknown compression format: ${dc.format}"
-            )
         }
       case pipe: PrepStepCfg.Pipe =>
         new ProcessPipeStep(pipe.command)

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala
@@ -37,7 +37,7 @@ class MetadataChain(datasetDir: Path) {
     val initialSummary = DatasetSummary(
       id = ds.id,
       kind = ds.kind,
-      datasetDependencies = ds.dependsOn.toSet,
+      dependencies = ds.dependsOn.toSet,
       vocab = ds.vocab,
       lastPulled = None,
       numRecords = 0,

diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala
@@ -116,7 +116,7 @@ class MetadataRepository(
     if (isRemote(id))
       List.empty
     else
-      getDatasetSummary(id).datasetDependencies.toList
+      getDatasetSummary(id).dependencies.toList
   }
 
   def getDatasetsInDependencyOrder(
@@ -212,7 +212,7 @@ class MetadataRepository(
     // Validate references
     val referencedBy = getAllDatasets()
       .map(getDatasetSummary)
-      .filter(_.datasetDependencies.contains(id))
+      .filter(_.dependencies.contains(id))
 
     if (referencedBy.nonEmpty)
       throw new DanglingReferenceException(referencedBy.map(_.id), id)

diff --git a/core.manifests b/core.manifests
+1 −1		src/main/scala/dev/kamu/core/manifests/DatasetSummary.scala
+18 −17		src/main/scala/dev/kamu/core/manifests/OpenDataFabric.scala
+16 −9		src/main/scala/dev/kamu/core/manifests/parsing/pureconfig/yaml/defaults/package.scala
+4 −4		src/test/scala/dev/kamu/core/manifests/UtilsSpec.scala