From 81ac05506b13458daa77b494641155be2164646b Mon Sep 17 00:00:00 2001 From: Sergii Mikhtoniuk Date: Sat, 22 Aug 2020 13:09:13 -0700 Subject: [PATCH] Update to new ODF schema --- CHANGELOG.md | 4 ++++ .../kamu/cli/commands/AddInteractiveCommand.scala | 7 +++++-- .../kamu/cli/commands/DependencyGraphCommand.scala | 2 +- .../dev/kamu/cli/ingest/fetch/SourceFactory.scala | 8 ++++---- .../dev/kamu/cli/ingest/prep/PrepStepFactory.scala | 12 ++++-------- .../scala/dev/kamu/cli/metadata/MetadataChain.scala | 2 +- .../dev/kamu/cli/metadata/MetadataRepository.scala | 4 ++-- core.manifests | 2 +- 8 files changed, 22 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6283c85482..147f0c3c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.23.0] - 2020-08-22 +### Changed +- Updated to latest ODF schema + ## [0.22.0] - 2020-07-15 ### Added - `#26`: Follow redirects when fetching data from URL diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala b/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala index eb25d1e696..6a2179393a 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/commands/AddInteractiveCommand.scala @@ -76,9 +76,12 @@ class AddInteractiveCommand( "Compression", "What's the compression format?", Seq("zip", "gzip") - ) + ) match { + case "zip" => CompressionFormat.Zip + case "gzip" => CompressionFormat.Gzip + } - val subPath = if (Seq("zip").contains(compression)) { + val subPath = if (compression == CompressionFormat.Zip) { inputOptional( "Sub-path", "If this archive can contain multiple files - specify the path regex to " + diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala b/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala index a0ca0d52d7..301cf4c957 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/commands/DependencyGraphCommand.scala @@ -30,7 +30,7 @@ class DependencyGraphCommand( datasets.foreach( ds => - ds.datasetDependencies + ds.dependencies .foreach(d => println(s"${quote(d)} -> ${quote(ds.id)};")) ) diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala index 82c81e935e..0ab94dae9e 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/fetch/SourceFactory.scala @@ -103,14 +103,14 @@ class SourceFactory(systemClock: Clock) { .toVector val orderBy = kind.order.getOrElse( - if (kind.eventTime.isDefined) SourceOrdering.ByEventTime() - else SourceOrdering.ByName() + if (kind.eventTime.isDefined) SourceOrdering.ByEventTime + else SourceOrdering.ByName ) val sorted = orderBy match { - case SourceOrdering.ByName() => + case SourceOrdering.ByName => globbed.sortBy(_.path.getFileName.toString.toLowerCase()) - case SourceOrdering.ByEventTime() => + case SourceOrdering.ByEventTime => globbed.sortBy(eventTimeSource.getEventTime) } diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala index 304ff91057..d9506c6b75 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/ingest/prep/PrepStepFactory.scala @@ -8,7 +8,7 @@ package dev.kamu.cli.ingest.prep -import dev.kamu.core.manifests.{PrepStep => PrepStepCfg} +import dev.kamu.core.manifests.{PrepStep => PrepStepCfg, CompressionFormat} import org.apache.logging.log4j.LogManager class PrepStepFactory() { @@ -19,17 +19,13 @@ class PrepStepFactory() { ): PrepStep = { config match { case dc: PrepStepCfg.Decompress => - dc.format.toLowerCase match { - case "gzip" => + dc.format match { + case CompressionFormat.Gzip => logger.debug("Extracting gzip") new DecompressGZIPStep() - case "zip" => + case CompressionFormat.Zip => logger.debug("Extracting zip") new DecompressZIPStep(dc) - case _ => - throw new NotImplementedError( - s"Unknown compression format: ${dc.format}" - ) } case pipe: PrepStepCfg.Pipe => new ProcessPipeStep(pipe.command) diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala index 2b1f528592..37434a3b3b 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataChain.scala @@ -37,7 +37,7 @@ class MetadataChain(datasetDir: Path) { val initialSummary = DatasetSummary( id = ds.id, kind = ds.kind, - datasetDependencies = ds.dependsOn.toSet, + dependencies = ds.dependsOn.toSet, vocab = ds.vocab, lastPulled = None, numRecords = 0, diff --git a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala index 0afe0b9c96..7e404c5d4c 100644 --- a/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala +++ b/core.coordinator/src/main/scala/dev/kamu/cli/metadata/MetadataRepository.scala @@ -116,7 +116,7 @@ class MetadataRepository( if (isRemote(id)) List.empty else - getDatasetSummary(id).datasetDependencies.toList + getDatasetSummary(id).dependencies.toList } def getDatasetsInDependencyOrder( @@ -212,7 +212,7 @@ class MetadataRepository( // Validate references val referencedBy = getAllDatasets() .map(getDatasetSummary) - .filter(_.datasetDependencies.contains(id)) + .filter(_.dependencies.contains(id)) if (referencedBy.nonEmpty) throw new DanglingReferenceException(referencedBy.map(_.id), id) diff --git a/core.manifests b/core.manifests index 8dd114fa2d..7dc52b3278 160000 --- a/core.manifests +++ b/core.manifests @@ -1 +1 @@ -Subproject commit 8dd114fa2ddb8ffd9567182c937135ce8f0ba8ac +Subproject commit 7dc52b327893d08e86df4fb1d30896725052f685