diff --git a/.env b/.env new file mode 100644 index 00000000..768845a4 --- /dev/null +++ b/.env @@ -0,0 +1,17 @@ +######################################## +# Adopt authentication for RabbitMQ +######################################## +RABBIT_MQ_USER=rabbituser +RABBIT_MQ_PASSWORD=rabbitpasswd +######################################## +# Only edit the following lines if you +# want to update service versions. +######################################## +METASTORE_VERSION=v1.4.0 +FRONTEND_COLLECTION_VERSION=metastore-v1.0.0 +INDEXING_SERVICE_VERSION=v1.0.1 +ELASTICSEARCH_VERSION=8.11.1 +######################################## +# Don't edit following lines +######################################## +PREFIX4DOCKER=ghcr.io/kit-data-manager diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 389bd377..93c88592 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,8 @@ updates: schedule: interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "development" + schedule: + interval: "weekly" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f1bf81b3..d8c9d80a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,11 +38,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -53,7 +53,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -67,4 +67,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f152a11c..5d93bbc0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,94 +1,118 @@ +# Create and publish a Docker image on github +name: Create and publish a Docker image -name: Docker - +# Configures this workflow to run every time a change +# is pushed to the 'main' branch. on: push: - # Publish `master` as Docker `latest` image. + # Publish `main` as Docker `latest` image. branches: - - master, main + - main # Publish `v1.2.3` tags as releases. tags: - v* + - '*-v*' - # Run tests for any PRs. - pull_request: - +# Defines two custom environment variables for the workflow. +# These are used for the Container registry domain, and a +# name for the Docker image that this workflow builds. env: - # TODO: Change variable to your image's name. - IMAGE_NAME: metastore2 + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} +# Two jobs for creating and pushing Docker image +# - build-and-push-image -> triggered by commits on main and tagging with semantic version (e.g.: v1.2.3) +# - build-and-push-image-of-branch -> triggered by tags matching '*-v*' (e.g.: Version_1-v1.2.3) jobs: - # Run tests. - # See also https://docs.docker.com/docker-hub/builds/automated-testing/ - test: + build-and-push-image: runs-on: ubuntu-latest - + if: ${{ contains(github.ref_name, '-') == failure() }} + # Sets the permissions granted to the `GITHUB_TOKEN` + # for the actions in this job. + permissions: + contents: read + packages: write + # steps: - - uses: actions/checkout@v2 - - - name: Run tests - run: | - ./gradlew jacocoTestReport -# docker build . --file Dockerfile - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - # Ensure test job passes before pushing image. - needs: test - + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container + # registry using the account and password that will publish the packages. + # Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) + # to extract tags and labels that will be applied to the specified image. + # The `id` "meta" allows the output of this step to be referenced in a + # subsequent step. The `images` value provides the base name for the tags + # and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-and-push-image-of-branch: runs-on: ubuntu-latest - if: github.event_name == 'push' - + if: contains(github.ref_name, '-') + # Sets the permissions granted to the `GITHUB_TOKEN` + # for the actions in this job. + permissions: + contents: read + packages: write + # steps: - - uses: actions/checkout@v2 - - - name: Build image + - name: Split first part + env: + TAG: ${{ github.ref_name }} + id: split + run: echo "branch=${TAG%-v*}" >> $GITHUB_OUTPUT + - name: Test variable run: | - docker build . --file Dockerfile --tag $IMAGE_NAME - - name: Log into registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - - - name: Push image - run: | - IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # Strip git ref prefix from version - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - # Strip "v" prefix from tag name - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - # Use Docker `latest` tag convention - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + echo ${{ steps.split.outputs.branch }} + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container + # registry using the account and password that will publish the packages. + # Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@v3 with: - # list of Docker images to use as base name for tags - images: | - kitdm/metastore2 - # add git short SHA as Docker tag - tag-sha: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) + # to extract tags and labels that will be applied to the specified image. + # The `id` "meta" allows the output of this step to be referenced in a + # subsequent step. The `images` value provides the base name for the tags + # and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta-branch + uses: docker/metadata-action@v5 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push to Docker Hub - uses: docker/build-push-action@v2 + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{steps.split.outputs.branch}} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@v5 with: context: . push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} \ No newline at end of file + tags: ${{ steps.meta-branch.outputs.tags }} + labels: ${{ steps.meta-branch.outputs.labels }} + diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index a2d373d0..01724ff7 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -11,7 +11,7 @@ on: env: # JDK version used for building jar file - currentBuildVersion: 11 + currentBuildVersion: 17 jobs: build: runs-on: ${{ matrix.operating-system }} @@ -19,12 +19,12 @@ jobs: matrix: operating-system: [ubuntu-latest, macOS-latest, windows-latest] # Use both LTS releases and latest one for tests - jdk: [ 11, 17 ] + jdk: [ 17, 19 ] steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up OpenJDK version ... - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: ${{ matrix.jdk }} @@ -48,9 +48,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up OpenJDK version ... - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: ${{ env.currentBuildVersion }} @@ -59,6 +59,6 @@ jobs: - name: Build with Gradle (JDK ${{ env.currentBuildVersion }}) run: ./gradlew clean check jacocoTestReport - name: Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: files: ./build/reports/jacoco/test/jacocoTestReport.xml #optional diff --git a/.gitignore b/.gitignore index 1be88301..b712a56f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ nbproject build config nbproject + +# Ignore VS Code +bin/ +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c354f66e..a8f46064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,121 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed + +### Fixed + +## [1.4.0] - 2023-12-19 +### Security + +### Added +- Add internal landing page +- Add configuration for redirecting landing page also to external site. +- Enable authentication for RabbitMQ (#430) +- +### Fixed +- Filter for rescourceId AND schemaId instead of OR (#401) + +### Changed +- Use softlink to jar file in start script. +- Refine return messages (#309, #402, #403) +- Check usage of schema before deleting (# +- Metadata schema identifiers support lowercase only +#### Github Actions +- Bump actions/checkout from 3 to 4 +- Bump actions/setup-java from 3 to 4 +- Bump crazy-max/ghaction-docker-meta from 4 to 5 +- Bump docker/build-push-action from 4 to 5 +- Bump docker/login-action from 2 to 3 +- Bump docker/setup-buildx-action from 2 to 3 +- Bump docker/setup-qemu-action from 2 to 3 +- Bump github/codeql-acion from 2 o 3 +- Bump gradle from 7.6.1 to 8.2.1 +- Bump org.mockito:mockito-core from 5.4.0 to 5.7.0 +#### Plugins +- Bump io.freefair.lombok from 8.1.0 to 8.4 +- Bump io.freefair.maven-publish-java from 8.1.0 to 8.4 +- Bump org.owasp.dependencycheck from 8.3.1 to 9.0.7 +- Bump io.spring.dependency-management from 1.1.0 to 1.1.4 +#### Libs +- Bump edu.kit.datamanager:indexing-service from 0.9.0 to 1.0.0 +- Bump com.networknt:json-schema-validator from 1.0.88 to 1.1.0 +- Bump com.networknt:json-schema-validator from 1.0.87 to 1.0.88 +- Bump com.google.errorprone:error_prone_core from 2.20.0 to 2.23.0 +- Bump com.h2database:h2 from 2.1.214 to 2.2.224 +- Bump com.networknt:json-schema-validator from 1.0.85 to 1.1.0 +- Bump commons-io:commons-io from 2.13.0 to 2.15.1 +- Bump org.apache.tika:tika-core from 2.8.0 to 2.9.1 +- Bump org.javers:javers-core from 7.0.0 to 7.3.6 +- Bump org.javers:javers-spring-boot-starter-sql from 7.0.0 to 7.3.6 +- Bump org.postgresql:postgresql from 42.6.0 to 42.7.1 +- Bump org.springdoc:springdoc-openapi-starter-common from 2.1.0 to 2.3.0 +- Bump org.springdoc:springdoc-openapi-starter-webmvc-api from 2.1.0 to 2.3.0 +- Bump org.springdoc:springdoc-openapi-starter-webmvc-ui from 2.1.0 to 2.3.0 +- Bump org.springframework:spring-messaging from 6.0.2 to 6.1.1 +- Bump org.springframework:spring-cloud-gateway-mvc from 4.0.6 to 4.1.0 +- Bump org.springframework:spring-cloud-starter-config from 4.0.3 to 4.1.0 +- Bump org.springframework:spring-cloud-starter-netflix-eureka-client from 4.0.2 to 4.1.0 +- Bump org.springframework.boot from 3.1.0 to 3.2.0 +- Bump org.springframework.data:spring-data-elasticsearch from 5.1.0 to 5.2.1 +- Bump org.apache.commons:commons-text from 1.10.0 to 1.11.0 + +## [1.3.0] - 2023-07-07 +### Security + +### Added +- Add context path 'metastore' as default. +- Add support for JSON schema 2020-12. + +### Fixed +- Fixed: allowing empty SID in ACLs. + +### Changed +- Bump JDK from 8 to 17. +- Bump actions/setup-java from 2 to 3 +- Bump com.networknt:json-schema-validator from 1.0.84 to 1.0.85 +- Bump crazy-max/ghaction-docker-meta from 1 to 4 +- Bump docker/build-push-action from 2 to 4 +- Bump docker/setup-buildx-action from 1 to 2 +- Bump edu.kit.datamanager:repo-core from 1.1.2 to 1.2.1 +- Bump edu.kit.datamanager:service-base from 1.1.1 to 1.2.0 +- Bump io.freefair.lombok from 8.0.1 to 8.1.0 +- Bump javersVersion from 6.14.0 to 7.0.0 +- Bump org.springframework:spring-messaging from 5.3.26 to 5.3.27 +- Bump org.springframework.boot from 2.7.10 to 3.1.0 +- Bump org.springframework.cloud:spring-cloud-starter-config from 3.1.7 to 4.0.3 +- Bump org.springframework.cloud:spring-cloud-starter-netflix-eureka-client from 3.1.6 to 4.0.2 +- Bump org.springframework.data:spring-data-elasticsearch from 4.4.13 to 5.1.0 +- Bump org.springframework.restdocs:spring-restdocs-mockmvc from 2.0.7.RELEASE to 3.0.0 +- Bump springDocVersion from 1.7.0 to 2.1.0 + +## [1.2.3] - 2023-04-13 +### Added +- Add CITATION.cff +- Allow cli arguments for start script. +- Enable configuration to organize internal storage of metadata documents (https://github.com/kit-data-manager/metastore2/issues/241) ### Fixed +- Providing (invalid) version number while updating schema document may break old versions. (https://github.com/kit-data-manager/metastore2/issues/245) +- Trace log may slow down service. (https://github.com/kit-data-manager/metastore2/issues/233) +- Calling REST-Endpoint for UI fails if no page information is provided. (https://github.com/kit-data-manager/metastore2/issues/264) +- Problem running MetaStore standalone in a docker container. (https://github.com/kit-data-manager/metastore2/issues/270) + +### Changed +- Update several badges +- Bump com.networknt:json-schema-validator from 1.0.78 to 1.0.79 +- Bump edu.kit.datamanager:repo-core from 1.1.1 to 1.1.2 +- Bump edu.kit.datamanager:service-base from 1.1.0 to 1.1.1 +- Bump gradle from 7.6 to 7.6.1. +- Bump io.freefair.maven-publish-java from 6.6.3 to 8.0.1 +- Bump io.freefair.lombok from 6.6.3 to 8.0.1 +- Bump org.mockito:mockito-core from 5.1.1 to 5.3.0 +- Bump org.owasp.dependencycheck from 8.1.0 to 8.2.1 +- Bump org.postgresql:postgresql from 42.5.4 to 42.6.0 +- Bump org.springframework.boot from 2.7.9 to 2.7.10 +- Bump org.springframework.cloud:spring-cloud-starter-config from 3.1.5 to 3.1.6. +- Bump org.springframework.data:spring-data-elasticsearch from 4.4.8 to 4.4.10. +- Bump org.springframework:spring-messaging from 5.3.25 to 5.3.26 +- Bump springDocVersion from 1.6.14 to 1.7.0 ## [1.2.2] - 2023-02-28 ### Fixed @@ -54,6 +167,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add actuator endpoints for info and health (https://github.com/kit-data-manager/metastore2/issues/184) - Add spring-data-elasticsearch 4.4.7 +### Fixed +- Invalid input for resource identifier causes NPE (https://github.com/kit-data-manager/metastore2/issues/198) +- Hibernate validation was not enabled by default. (https://github.com/kit-data-manager/metastore2/issues/191) +- Check metadata directory for valid entry during startup (https://github.com/kit-data-manager/metastore2/issues/185) + ### Changed - Bump commons-text from 1.9 to 1.10.0 - Bump gradle from 7.5.1 to 7.6 @@ -80,11 +198,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bump tika-core from 1.2.7 to 2.6.0 - Bump xercesImpl from 2.12.1 to 2.12.2 -### Fixed -- Invalid input for resource identifier causes NPE (https://github.com/kit-data-manager/metastore2/issues/198) -- Hibernate validation was not enabled by default. (https://github.com/kit-data-manager/metastore2/issues/191) -- Check metadata directory for valid entry during startup (https://github.com/kit-data-manager/metastore2/issues/185) - ## [1.1.0] - 2022-10-17 ### Security - Switch to 'eclipse-temurin' for docker due to end of support for 'openjdk'. @@ -119,10 +232,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Please migrate your database if you want to update MetaStore while using h2! See: https://h2database.com/html/migration-to-v2.html -### Changed -- Update to service-base 1.0.3 -- Changed some labels in GUI - ### Fixed - Docker: Add support for M1 chip architecture (https://github.com/kit-data-manager/metastore2/issues/107) - Access public documents of other users is broken.(https://github.com/kit-data-manager/metastore2/issues/100) @@ -131,6 +240,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix bugs using Swagger UI for REST calls. - Fix typos in documentation. +### Changed +- Update to service-base 1.0.3 +- Changed some labels in GUI + ## [1.0.0] - 2022-03-29 ### Added - Finalized version of MetaStoreGui @@ -138,13 +251,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allow also IDs for metadata documents (https://github.com/kit-data-manager/metastore2/issues/76) - Access filter for monitoring. -### Changed -- Update to repo-core 1.0.2 -- Update to service-base 1.0.1 -- Update to postgresql 42.2.25 -- Downgrade library due to some issues regarding validation - - json-schema-validator 1.0.64. -> 1.0.59 (https://github.com/kit-data-manager/metastore2/issues/77) - ### Fixed - Fix bug listing resources without proper authorization (https://github.com/kit-data-manager/metastore2/issues/71) - Fix bug listing all metadata documents related to a specific schema @@ -154,6 +260,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Check ACLs while creating/updating records (https://github.com/kit-data-manager/metastore2/issues/39) - Added missing spaces to swagger-ui. +### Changed +- Update to repo-core 1.0.2 +- Update to service-base 1.0.1 +- Update to postgresql 42.2.25 +- Downgrade library due to some issues regarding validation + - json-schema-validator 1.0.64. -> 1.0.59 (https://github.com/kit-data-manager/metastore2/issues/77) + ## [0.3.7] - 2022-01-11 ### Added - First version of GUI for MetaStore. @@ -215,15 +328,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Metadata (schema) records now also versioned. - OAI PMH protocol added (https://github.com/kit-data-manager/metastore2/issues/6) - Customization is enabled even if framework will be started via docker (https://github.com/kit-data-manager/metastore2/issues/41) -### Changed -- Metadata are now linked to specific version of a schema. (https://github.com/kit-data-manager/metastore2/issues/30) -- Attribute 'locked' from MetadataSchemaRecord changed to 'doNotSync' (https://github.com/kit-data-manager/metastore2/issues/37) -- Change in related schema and data (add identifier type to identifier) -- Store all identifiers as global identifiers (type INTERNAL -> type URL) -- For registering a schema mimetype is no longer mandatory. -- Switch to gradle version 7.2 -- Update to Spring Boot 2.4.10 -- Update to service-base version 0.3.0 + ### Fixed - Filtering metadata documents by resourceId, schemaId - Filtering schema documents by mimetype @@ -234,6 +339,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add hash of schema documents to record. (https://github.com/kit-data-manager/metastore2/issues/38) - Drop tables at startup (default). +### Changed +- Metadata are now linked to specific version of a schema. (https://github.com/kit-data-manager/metastore2/issues/30) +- Attribute 'locked' from MetadataSchemaRecord changed to 'doNotSync' (https://github.com/kit-data-manager/metastore2/issues/37) +- Change in related schema and data (add identifier type to identifier) +- Store all identifiers as global identifiers (type INTERNAL -> type URL) +- For registering a schema mimetype is no longer mandatory. +- Switch to gradle version 7.2 +- Update to Spring Boot 2.4.10 +- Update to service-base version 0.3.0 + ## [0.2.4] - date 2020-12-16 ### Added - Support for messaging @@ -280,7 +395,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Registry for XSD files and support for XML metadata -[Unreleased]: https://github.com/kit-data-manager/metastore2/compare/v1.2.2...HEAD +[Unreleased]: https://github.com/kit-data-manager/metastore2/compare/v1.4.0...HEAD +[1.4.0]: https://github.com/kit-data-manager/metastore2/compare/v1.3.0...v1.4.0 +[1.3.0]: https://github.com/kit-data-manager/metastore2/compare/v1.2.3...v1.3.0 +[1.2.3]: https://github.com/kit-data-manager/metastore2/compare/v1.2.2...v1.2.3 [1.2.2]: https://github.com/kit-data-manager/metastore2/compare/v1.2.1...v1.2.2 [1.2.1]: https://github.com/kit-data-manager/metastore2/compare/v1.2.0...v1.2.1 [1.2.0]: https://github.com/kit-data-manager/metastore2/compare/v1.1.0...v1.2.0 diff --git a/INTERFACE.md b/INTERFACE.md new file mode 100644 index 00000000..2e3916a9 --- /dev/null +++ b/INTERFACE.md @@ -0,0 +1,132 @@ +--- +filename: INTERFACES.md +description: MetaStore +idea: One such file per repository (services, plugins, frontends, ...) as a proposal for DEM developers. +... + +# Interfaces Overview for MetaStore + +This document aims to answer questions on how to configure external dependencies and which public interfaces are offered by MetaStore in a comprehensive way. It is meant to be used for getting an overview and guidance in addition to the official documentation, which is available at the official [MetaStore Web page](https://kit-data-manager.github.io/webpage/metastore/).How can this software be connected with other software? This is the question this document aims to answer. + +> ℹī¸ **Note:** +> This document applies to the MetaStore version it is shipped with. If you have a specific version running, please refer to `INTERFACE.md` of this particular release. + +## TOC + +- [Interfaces Overview for MetaStore](#interfaces-overview-for-metastore) + * [External Dependencies](#external-dependencies-) 📤 + + [Relational Database (mandatory)](#relational-database-mandatory-) ⛁ + + [Local Filesystem (mandatory)](#local-filesystem-mandatory-) 📂 + + [Messaging (optional)](#messaging-optional-) đŸ’Ŧ + + [Enhanced Search (optional)](#enhanced-search-optional-) 🔍 + + [Access Control (optional)](#access-control-optional-) 🔐 + * [Public Interfaces](#public-interfaces-) đŸ“Ĩ + + [HTTP / REST](#http--rest) + + [Elasticsearch Proxy](#elasticsearch-proxy-) 🔍 + + [OAI-PMH](#oai-pmh) + + [Digital Object Interface Protocol (DOIP)](#digital-object-interface-protocol-doip) +## External Dependencies 📤 + +External dependencies are third-party services that are required for MetaStore to work properly or that can be added optionally to provide additional functionality. Typically, external dependencies require additional software to be installed and configured, before they can be included in the MetaStore configuration, which is typically done via the main configuration file `application.properties`. If you do not want to lose your default settings, we recommend that you make a copy of "application.properties" and move it to the "config" subfolder. Remove all properties you want to keep from the new file. **All properties in "config/application.properties" override the settings in "application.properties".** + + +## External Dependencies 📤 + +External dependencies are third-party services that are required for MetaStore to work properly or that can be added optionally to provide additional functionality. Typically, external dependencies require +additional software to be installed and configured, before they can be included in the MetaStore configuration, which is should be done via the configuration file `config/application.properties`. + +### Relational Database (mandatory) ⛁ +A relational database is required by MetaStore to store administrative metadata for metadata documents/schemas. If not configured properly, MetaStore will fail to start. + +#### Configuration ⚙ī¸ + - H2 In-Memory (driver included, used for **testing only**, **not** recommended for **production**) [Example](https://github.com/kit-data-manager/metastore2/blob/v1.3.0/src/test/resources/test-config/application-test.properties#L38-L44) + - H2 File-Based (driver included, used for basic Docker setup or for 'quick and dirty' tests, **not** recommended for **production**) [Example](https://github.com/kit-data-manager/metastore2/blob/v1.3.0/settings/application-docker.properties#L106-L113) + - Postgres (driver included, **requires a running PostgreSQL server**, **recommended for production**) [PostgreSQL](https://www.postgresql.org/), [Example](https://github.com/kit-data-manager/metastore2/blob/v1.3.0/settings/application-postgres.properties#L106-L116) + +> ℹī¸ **Note:** +> Other relational databases, such as MariaDB, SQLite, or Oracle, may also work, but require additional steps. To allow MetaStore to connect, the source code repository must be cloned, an appropriate JDBC driver has to be added to `build.gradle` and MetaStore has to be compiled. Proper JDBC drivers are typically provided on the database's web page. Afterwards, the database can be configured in `config/application.properties` similar to PostgreSQL but with database-specific property naming. Please refer to the driver documentation for details. + +### Local Filesystem (mandatory) 📂 +MetaStore requires access to the local file system in order to store and manage uploaded metadata (schema) documents. MetaStore requires access to two folders, which can be located on the local hard drive or mounted via NFS. By default, two subfolders named 'metadata' and 'schema' are created in the installation directory. + + +#### Configuration ⚙ī¸ + - see `application.properties` + - [Configure folders](https://github.com/kit-data-manager/metastore2/blob/v1.3.0/settings/application-default.properties#L18-L22) + - [Configure folder structure for metadata documents](https://github.com/kit-data-manager/metastore2/blob/v1.3.0/settings/application-default.properties#L24-L42) + +> ℹī¸ **Note:** +> The file path to the folders have to start with three '/'. If you overwrite the default settings we recommend to create or edit 'config/application.properties'. e.g.: +> ``` title= config/application.properties +> metastore.schema.schemaFolder:file:///data/metastore/schema +> metastore.metadata.metadataFolder:file:///data/metastore/metadata +>``` +> **If the folders do not exist, they will be created.** + +### Messaging (optional) đŸ’Ŧ +AMQP-based messaging is an optional feature of MetaStore, which allows MetaStore to emit messages about creation, modification, and deletion events related to **metadata documents**. These messages can be received by registered consumers and processed in an asynchronous way. +#### Installation +- [Installing RabbitMQ](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/framework/setup-rabbitMq.html) +#### Configuration ⚙ī¸ + - [RabbitMQ](https://www.rabbitmq.com/) (dependencies included, serves as messaging distributor, requires a running RabbitMQ server) + - [Introduction Messaging for MetaStore](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/messaging/messaging-introduction.html) + - [Configuration Messaging for MetaStore](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/messaging/messaging-configuration.html) + - [Example](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#rabbitmq) + +### Enhanced Search (optional) 🔍 +By default, MetaStore offers basic search based on administrative metadata via RESTful API by certain query parameters. Optionally, enhanced search via a search index can be enabled and used for fine-grained and facetted search operations.Therefor all metadata documents were indexed using an [elasticsearch instance](https://www.elastic.co/de/elasticsearch/). (JSON metadata documents are provided by default. XML documents should be transformed to JSON) +#### Requirements +- [Messaging](#messaging-optional-) +- Indexing Service + - [Installation](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/framework/setup-indexing-service.html) + - [Configuration](https://github.com/kit-data-manager/indexing-service/blob/main/settings/application-default.properties#L12-L21) +- [Elasticsearch](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/framework/setup-elasticsearch.html) +Messaging is used to inform indexing service about any updates regarding metadata documents. Indexing service will fetch the document, transform it (if mapping available) and send it to elasticsearch for indexing. +#### Configuration ⚙ī¸ +- [Configuration](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#elasticsearch) + +### Access Control (optional) 🔐 +By default, MetaStore itself is open for all kinds of operations, i.e., read and write, where write access should be restricted on the user interface level, e.g., by a password-protected area for critical operations. Optionally, authentication and authorization via JSON Web Tokens (JWT) issued by a Keycloak instance, can be configured, which allows a fine-granulated access management on document level. + +#### Requirements +- [Keycloak](https://www.keycloak.org/) +#### Configuration ⚙ī¸ +- [Setup MetaStore](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#keycloak) + +## Public Interfaces đŸ“Ĩ +Public Interfaces are used to access MetaStore in order to obtain its contents, typically this happens via HTTP/REST. Depending on the interface, special clients or protocols must be used to access a specific public interface. + + +Interfaces which trigger functionality or return information **on request**. + +### HTTP / REST +A REST interface which allows to access the service functionality, like creating/registering, updating and validating metadata documents/schemas on request. +#### Documentation 📖 + - [OpenAPI](https://kit-data-manager.github.io/webpage/metastore/documentation/api-docs.html) / or via running instance, as described in the [Readme](https://github.com/kit-data-manager/metastore2#first-steps-using-framework). + - [Usage with Examples](https://kit-data-manager.github.io/webpage/metastore/documentation/REST/index.html) + +#### Application Examples 📋 +- [Frontend Collection](https://github.com/kit-data-manager/frontend-collection) +- [Metadata Hub](https://git.rwth-aachen.de/nfdi4ing/s-3/s-3-3/metadatahub) + +### Elasticsearch Proxy 🔍 +If [Enhanced Search](#enhanced-search-optional-) is enabled, an additional REST endpoint becomes available, which allows to tunnel search queries to the underlying Elasticsearch instance. The advantage for proxying Elasticsearch access is, that access restrictions enabled via [Access Control](#access-control-optional-) are included in the query such that only results accessible by the caller are returned. + +#### Documentation 📖 + - [Search Configuration](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/search-configuration.html) + +#### Application Examples 📋 +- [Frontend Collection](https://github.com/kit-data-manager/frontend-collection) +- [Metadata Hub](https://git.rwth-aachen.de/nfdi4ing/s-3/s-3-3/metadatahub) + +### OAI-PMH +[OAI-PMH](https://www.openarchives.org/pmh/) is a standardized harvesting protocol that allows to build up external search indices that can be kept up to data by regular harvesting changes from an OAI-PMH source. For MetaStore only XML metadata documents are supported. +#### Documentation 📖 + - [Configuration](https://kit-data-manager.github.io/webpage/metastore/documentation/installation/setup-metastore.html#oai-pmh) + +### ~~Digital Object Interface Protocol (DOIP)~~ +[DOIP](https://www.dona.net/sites/default/files/2018-11/DOIPv2Spec_1.pdf) is a novel protocol to provide generic access to digital resources. Instead of using HTTP-based communication, the protocol acts directly on top of TCP/IP and is therefore mainly relevant for special ecosystems. +**Not finalized yet!** + +#### Documentation 📖 +Not yet available! diff --git a/README.md b/README.md index 1eec77d9..93f28af5 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,18 @@ [![Codecov](https://codecov.io/gh/kit-data-manager/metastore2/graph/badge.svg)](https://codecov.io/gh/kit-data-manager/metastore2) [![CodeQL](https://github.com/kit-data-manager/metastore2/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/kit-data-manager/metastore2/actions/workflows/codeql-analysis.yml) ![License](https://img.shields.io/github/license/kit-data-manager/metastore2.svg) -[![Docker Build Status](https://img.shields.io/docker/automated/kitdm/metastore2)](https://hub.docker.com/r/kitdm/metastore2/tags) -[![Docker Image Version](https://img.shields.io/docker/v/kitdm/metastore2)](https://hub.docker.com/r/kitdm/metastore2/tags) -[![Docker Pulls](https://img.shields.io/docker/pulls/kitdm/metastore2)](https://hub.docker.com/r/kitdm/metastore2/tags) +[![docker]](https://github.com/kit-data-manager/metastore2/pkgs/container/metastore2) +[![currentVersion]](https://github.com/kit-data-manager/metastore2/pkgs/container/metastore2) +[![size]](https://github.com/kit-data-manager/metastore2/pkgs/container/metastore2) -General purpose metadata repository and schema registry service. + +[docker]: +[currentVersion]: +[size]: + +MetaStore is a research data repository software for storing metadata documents and schemas. +Quality and consistency are ensured by associating and validating each document against a schema. +It supports JSON and XML. It allows you to - register an (XML/JSON) schema @@ -20,19 +27,19 @@ It allows you to ## Installation There are three ways to install metaStore2 as a microservice: -- [Using](#Installation-via-DockerHub) the image available via [DockerHub](https://hub.docker.com/r/kitdm/) (***recommended***) +- [Using](#Installation-via-GitHub-Packages) the image available via [GitHub Packages](https://github.com/orgs/kit-data-manager/packages?repo_name=metastore2) (***recommended***) - [Building](#Build-docker-container-locally) docker image locally - [Building](#Build-and-run-locally) and running locally -## Installation via DockerHub +## Installation via GitHub Packages ### Prerequisites In order to run this microservice via docker you'll need: * [Docker](https://www.docker.com/) ### Installation -Typically, there is no need for locally building images as all version are accessible via DockerHub. -Have a look of available images and their tags [here](https://hub.docker.com/r/kitdm/metastore2) +Typically, there is no need for locally building images as all version are accessible via GitHub Packages. +Have a look of available images and their tags [here](https://github.com/orgs/kit-data-manager/packages?repo_name=metastore2) Just follow instructions [below](#Build-docker-container). ## Build docker container locally @@ -57,12 +64,12 @@ user@localhost:/home/user/metastore2$ Now you'll have to create an image containing the microservice. This can be done via a script. On default the created images will be tagged as follows: -*'latest tag'-'actual date(yyyy-mm-dd)'* (e.g.: 0.1.1-2020-10-05) +*'latest tag'-'actual date(yyyy-mm-dd)'* (e.g.: 1.2.0-2023-06-27) ``` user@localhost:/home/user/metastore2$ bash docker/buildDocker.sh --------------------------------------------------------------------------- -Build docker container kitdm/metastore2:0.1.1-2020-10-05 +Build docker container ghcr.io/kit-data-manager/metastore2:1.2.0-2023-06-27 --------------------------------------------------------------------------- [...] --------------------------------------------------------------------------- @@ -75,9 +82,9 @@ user@localhost:/home/user/metastore2$ After building image you have to create (and start) a container for executing microservice: ``` # If you want to use a specific image you may list all possible tags first. -user@localhost:/home/user/metastore2$ docker images kitdm/metastore2 --format {{.Tag}} -0.1.1-2020-10-05 -user@localhost:/home/user/metastore2$ docker run -d -p8040:8040 --name metastore4docker kitdm/metastore2:0.1.1-2020-10-05 +user@localhost:/home/user/metastore2$ docker images ghcr.io/kit-data-manager/metastore2 --format {{.Tag}} +1.2.0-2023-06-27 +user@localhost:/home/user/metastore2$ docker run -d -p8040:8040 --name metastore4docker ghcr.io/kit-data-manager/metastore2:1.2.0-2023-06-27 57c973e7092bfc3778569f90632d60775dfecd12352f13a4fd2fdf4270865286 user@localhost:/home/user/metastore2$ ``` @@ -92,7 +99,7 @@ Therefor you have to provide an additional flag to the command mentioned before: user@localhost:/home/user/metastore2$ mkdir config # Place your own 'application.properties' inside the config directory # Create/run container -user@localhost:/home/user/metastore2$ docker run -d -p8040:8040 -v `pwd`/config:/spring/metastore2/config --name metastore4docker kitdm/metastore2:0.1.1-2020-10-05 +user@localhost:/home/user/metastore2$ docker run -d -p8040:8040 -v `pwd`/config:/spring/metastore2/config --name metastore4docker ghcr.io/kit-data-manager/metastore2:1.2.0-2023-06-27 57c973e7092bfc3778569f90632d60775dfecd12352f13a4fd2fdf4270865286 user@localhost:/home/user/metastore2$ ``` @@ -113,7 +120,7 @@ user@localhost:/home/user/metastore2$ docker start metastore4docker ### Prerequisites In order to run this microservice via docker you'll need: -* [Java SE Development Kit >=8 and <=17](https://openjdk.java.net/) +* [Java SE Development Kit >= 17](https://openjdk.java.net/) * [git](https://git-scm.com/) ### Installation @@ -168,7 +175,7 @@ https://kit-data-manager.github.io/webpage/metastore/documentation/REST/index.ht If you want to use a script for doing so please refer to -http://metastore.docker:8040/swagger-ui.html +http://metastore.docker:8040/metastore/swagger-ui.html in order to see available RESTful endpoints and their documentation. Furthermore, you can use this Web interface to test single API calls in order to get familiar with the @@ -198,7 +205,7 @@ user@localhost:/home/user/metastore2$ ## First steps using MetaStore standalone If you're using MetaStore without the whole framework the service is reachable via -http://localhost:8040/.... +http://localhost:8040/metastore/.... ## Setup for production mode :WARNING: If you want to use the service in production mode you have modify your configuration (application.properties). diff --git a/bin/main/static/jsonSchemas/schemaRecord.json b/bin/main/static/jsonSchemas/schemaRecord.json new file mode 100644 index 00000000..44fa9560 --- /dev/null +++ b/bin/main/static/jsonSchemas/schemaRecord.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "http://www.example.org/schema/json", + "type": "object", + "properties": { + "schemaId": { + "type": "string", + "title": "Schema Record Identifier" + }, + "schemaVersion": { + "type": "string", + "title": "Schema Version" + }, + "mimeType": { + "type": "string", + "title": "MIME Type", + "default": "application/json", + "enum": ["application/json", "application/xml"] + }, + "type": { + "type": "string", + "title": "Type", + "default": "JSON", + "enum": ["JSON", "XML"] + }, + "createdAt": { + "type": "datetime", + "title": "Date Created" + }, + "lastUpdate": { + "type": "datetime", + "title": "Date Updated" + }, + "label": { + "type": "string", + "title": "Label" + }, + "definition": { + "type": "string", + "title": "Definition" + }, + "comment": { + "type": "string", + "title": "Comment" + }, + "acl": { + "type": "array", + "title": "ACL", + "items": { + "type": "object", + "properties": { + "sid": { + "type": "string", + "title": "sid" + }, + "permission": { + "type": "string", + "title": "Permission", + "default": "ADMINISTRATE", + "enum": ["NONE", "READ", "WRITE", "ADMINISTRATE"] + } + } + } + }, + "schemaDocumentUri": { + "type": "string", + "title": "Schema Document Uri" + }, + "schemaHash": { + "type": "string", + "title": "Schema Hash" + }, + "locked": { + "type": "boolean", + "default": "false", + "title": "Locked" + } + } +} \ No newline at end of file diff --git a/build.gradle b/build.gradle index 6b37f282..934479fb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,10 +1,11 @@ plugins { - id 'org.springframework.boot' version '2.7.9' - id 'io.spring.dependency-management' version '1.1.0' - id 'io.freefair.lombok' version '6.6.3' - id 'io.freefair.maven-publish-java' version '6.6.3' - id 'org.owasp.dependencycheck' version '8.1.2' + id 'org.springframework.boot' version '3.2.1' + id 'io.spring.dependency-management' version '1.1.4' + id 'io.freefair.lombok' version '8.4' + id 'io.freefair.maven-publish-java' version '8.4' + id 'org.owasp.dependencycheck' version '9.0.7' id 'org.asciidoctor.jvm.convert' version '3.3.2' + id 'net.ltgt.errorprone' version '3.1.0' id 'net.researchgate.release' version '3.0.2' id 'com.gorylenko.gradle-git-properties' version '2.4.1' id 'java' @@ -16,10 +17,10 @@ group = 'edu.kit.datamanager' ext { // versions of dependencies - springDocVersion = '1.6.15' - javersVersion = '6.12.0' + springDocVersion = '2.3.0' + javersVersion = '7.3.7' keycloakVersion = '19.0.0' - + errorproneVersion = '2.24.0' // directory for generated code snippets during tests snippetsDir = file("build/generated-snippets") } @@ -33,8 +34,10 @@ repositories { mavenCentral() } -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} if (System.getProperty('profile') == 'minimal') { println 'Using minimal profile for building ' + project.getName() @@ -46,8 +49,8 @@ if (System.getProperty('profile') == 'minimal') { dependencies { // Spring - implementation 'org.springframework:spring-messaging:5.3.26' - implementation 'org.springframework.cloud:spring-cloud-gateway-mvc:3.1.6' + implementation 'org.springframework:spring-messaging:6.1.1' + implementation 'org.springframework.cloud:spring-cloud-gateway-mvc:4.1.1' // Spring Boot implementation "org.springframework.boot:spring-boot-starter-data-rest" @@ -66,20 +69,13 @@ dependencies { // cloud support - implementation "org.springframework.cloud:spring-cloud-starter-config:3.1.6" - implementation "org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:3.1.5" + implementation "org.springframework.cloud:spring-cloud-starter-config:4.1.0" + implementation "org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:4.1.0" // springdoc - implementation "org.springdoc:springdoc-openapi-ui:${springDocVersion}" - implementation "org.springdoc:springdoc-openapi-data-rest:${springDocVersion}" - implementation "org.springdoc:springdoc-openapi-webmvc-core:${springDocVersion}" - - //Keycloak already imported in service-base -// implementation "org.keycloak:keycloak-spring-boot-starter:${keycloakVersion}" -// implementation "com.nimbusds:nimbus-jose-jwt:9.23" -// implementation "io.jsonwebtoken:jjwt-api:0.11.5" -// implementation "io.jsonwebtoken:jjwt-impl:0.11.5" -// implementation "io.jsonwebtoken:jjwt-jackson:0.11.5" + implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${springDocVersion}" + implementation "org.springdoc:springdoc-openapi-starter-common:${springDocVersion}" + implementation "org.springdoc:springdoc-openapi-starter-webmvc-api:${springDocVersion}" // CLI Parser implementation "com.beust:jcommander:1.82" @@ -88,7 +84,7 @@ dependencies { implementation "commons-configuration:commons-configuration:1.10" implementation "org.apache.commons:commons-collections4:4.4" // includes commons-lang3 - implementation "org.apache.commons:commons-text:1.10.0" + implementation "org.apache.commons:commons-text:1.11.0" // UI (website) implementation "org.springframework.boot:spring-boot-starter-thymeleaf" @@ -98,51 +94,63 @@ dependencies { implementation "org.javers:javers-core:${javersVersion}" // driver for postgres - implementation "org.postgresql:postgresql:42.6.0" + implementation "org.postgresql:postgresql:42.7.1" //driver for h2 - implementation "com.h2database:h2:2.1.214" + implementation "com.h2database:h2:2.2.224" // apache - implementation "commons-io:commons-io:2.11.0" - implementation "org.apache.tika:tika-core:2.7.0" + implementation "commons-io:commons-io:2.15.1" + implementation "org.apache.tika:tika-core:2.9.1" // JSON validator - implementation "com.networknt:json-schema-validator:1.0.78" + implementation "com.networknt:json-schema-validator:1.1.0" // XML validator // https://mvnrepository.com/artifact/xerces/xercesImpl implementation 'xerces:xercesImpl:2.12.2' // datamanager - implementation "edu.kit.datamanager:repo-core:1.1.2" - implementation "edu.kit.datamanager:service-base:1.1.1" + implementation "edu.kit.datamanager:repo-core:1.2.1" + implementation "edu.kit.datamanager:service-base:1.2.0" // elasticsearch (since service-base 1.1.0) - implementation "org.springframework.data:spring-data-elasticsearch:4.4.9" + implementation "org.springframework.data:spring-data-elasticsearch:5.2.1" // DOIP SDK - implementation "net.dona.doip:doip-sdk:2.1.0" + implementation "net.dona.doip:doip-sdk:2.2.0" runtimeOnly "org.apache.httpcomponents:httpclient:4.5.14" // Additional libraries for tests - testImplementation "org.springframework.restdocs:spring-restdocs-mockmvc:2.0.7.RELEASE" + testImplementation "org.springframework.restdocs:spring-restdocs-mockmvc:3.0.0" testImplementation "org.springframework.boot:spring-boot-starter-test" testImplementation "org.springframework:spring-test" testImplementation "org.springframework.security:spring-security-test" //Java 11 Support - testImplementation "org.mockito:mockito-core:5.2.0" + testImplementation "org.mockito:mockito-core:5.8.0" testImplementation "junit:junit:4.13.2" + testImplementation "com.github.stefanbirkner:system-lambda:1.2.1" + + // errorprone + errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}" } compileJava { - options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" +// options.errorprone.disableAllWarnings = true + options.errorprone.disableWarningsInGeneratedCode = true + options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" << "-Xmaxwarns" << "200" +} + +compileTestJava { + // disable errorprone for tests + options.errorprone.disableWarningsInGeneratedCode = true + options.errorprone.enabled = false // change it to true to enable } test { finalizedBy jacocoTestReport environment "spring.config.location", "classpath:/test-config/" - + maxHeapSize = "8192m" testLogging { outputs.upToDateWhen {false} @@ -157,7 +165,7 @@ tasks.withType(Test) { } jacoco { - toolVersion = "0.8.7" + toolVersion = "0.8.10" } import java.text.SimpleDateFormat @@ -202,7 +210,7 @@ bootJar { duplicatesStrategy = DuplicatesStrategy.EXCLUDE manifest { - attributes 'Main-Class': 'org.springframework.boot.loader.PropertiesLauncher' + attributes 'Main-Class': 'org.springframework.boot.loader.launch.PropertiesLauncher' } dependsOn asciidoctor from ("${asciidoctor.outputDir}") { diff --git a/build.sh b/build.sh index e4062030..63aedbef 100644 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ ################################################################################ function usage { ################################################################################ - echo "Script for creating metastore service." + echo "Script for creating $REPO_NAME service." echo "USAGE:" echo " $0 [/path/to/installation/dir]" echo "IMPORTANT: Please enter an empty or new directory as installation directory." @@ -87,7 +87,7 @@ testForCommands="chmod cp dirname find java javac mkdir git" for command in $testForCommands do - if ! type $command >> /dev/null; then + if ! type "$command" >> /dev/null; then echo "Error: command '$command' is not installed!" exit 1 fi @@ -98,11 +98,6 @@ done ################################################################################ ACTUAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -################################################################################ -# Check parameters -################################################################################ -checkParameters "$*" - ################################################################################ # Determine repo name ################################################################################ @@ -110,6 +105,11 @@ REPO_NAME=$(./gradlew -q printProjectName) # Use only last line REPO_NAME=${REPO_NAME##*$'\n'} +################################################################################ +# Check parameters +################################################################################ +checkParameters "$*" + printInfo "Build microservice of $REPO_NAME at '$INSTALLATION_DIRECTORY'" ################################################################################ @@ -150,6 +150,8 @@ cd "$INSTALLATION_DIRECTORY" || { echo "Failure changing to directory $INSTALLAT # Determine name of jar file. jarFile=($(ls $REPO_NAME*.jar)) +# Create soft link for jar file +ln -s ${jarFile[0]} $REPO_NAME.jar { echo "#!/bin/bash" @@ -167,7 +169,7 @@ jarFile=($(ls $REPO_NAME*.jar)) echo "################################################################################" echo "# Define jar file" echo "################################################################################" - echo "jarFile=${jarFile[0]}" + echo "jarFile=${REPO_NAME}.jar" echo " " echo "################################################################################" echo "# Determine directory of script." @@ -178,8 +180,9 @@ jarFile=($(ls $REPO_NAME*.jar)) echo "################################################################################" echo "# Start micro service" echo "################################################################################" - echo "java -cp \".:\$jarFile\" -Dloader.path=\"file://\$ACTUAL_DIR/\$jarFile,./lib/,.\" -jar \$jarFile" + echo "java -cp \".:\$jarFile\" -Dloader.path=\"file://\$ACTUAL_DIR/\$jarFile,./lib/,.\" -jar \$jarFile \$*" } > run.sh + # make script executable chmod 755 run.sh diff --git a/docker-compose.yml b/docker-compose.yml index af0060cd..3797c6dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,17 +12,18 @@ services: dps: ipv4_address: 172.0.0.10 my-apache: - build: https://github.com/kit-data-manager/frontend-collection.git#docker-metastore - image: my-apache + image: ${PREFIX4DOCKER}/frontend-collection-metastore:${FRONTEND_COLLECTION_VERSION} + container_name: my_apache ports: - "80:80" networks: - dps elasticsearch: - image: elasticsearch:7.9.3 + image: elasticsearch:${ELASTICSEARCH_VERSION} container_name: elastic4indexing environment: - discovery.type=single-node + - xpack.security.enabled=false # - logger.org.elasticsearch=ERROR - HOSTNAMES=elastic.docker healthcheck: @@ -43,14 +44,19 @@ services: container_name: rabbitmq4indexing environment: - HOSTNAMES=rabbitmq.docker + - RABBITMQ_DEFAULT_USER=${RABBIT_MQ_USER} + - RABBITMQ_DEFAULT_PASS=${RABBIT_MQ_PASSWORD} ports: - "5672:5672" - "15672:15672" networks: - dps indexing-service: - image: kitdm/indexing-service:v0.1.3 + image: ${PREFIX4DOCKER}/indexing-service:${INDEXING_SERVICE_VERSION} container_name: indexing4metastore + environment: + - REPO_MESSAGING_USERNAME=${RABBIT_MQ_USER} + - REPO_MESSAGING_PASSWORD=${RABBIT_MQ_PASSWORD} depends_on: rabbitmq: condition: service_started @@ -63,10 +69,14 @@ services: networks: - dps metastore: - image: kitdm/metastore2:v1.2.2 + image: ${PREFIX4DOCKER}/metastore2:${METASTORE_VERSION} container_name: metastore.docker environment: - HOSTNAMES=metastore.docker + - REPO_SEARCH_ENABLED=true + - REPO_MESSAGING_ENABLED=true + - REPO_MESSAGING_USERNAME=${RABBIT_MQ_USER} + - REPO_MESSAGING_PASSWORD=${RABBIT_MQ_PASSWORD} depends_on: rabbitmq: condition: service_started diff --git a/docker/buildDocker.sh b/docker/buildDocker.sh index df823738..f40dd08a 100644 --- a/docker/buildDocker.sh +++ b/docker/buildDocker.sh @@ -41,7 +41,7 @@ testForCommands="dirname date docker git" for command in $testForCommands do - if ! type $command >> /dev/null; then + if ! type "$command" >> /dev/null; then echo "Error: command '$command' is not installed!" exit 1 fi @@ -55,7 +55,7 @@ ACTUAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ################################################################################ # Determine repo name ################################################################################ -REPO_NAME=$($ACTUAL_DIR/../gradlew -q printProjectName) +REPO_NAME=$("$ACTUAL_DIR"/../gradlew -q printProjectName) # Use only last line REPO_NAME=${REPO_NAME##*$'\n'} @@ -80,18 +80,18 @@ else TAG_NAME=$LAST_TAG-$(date -u +%Y-%m-%d) fi -cd $ACTUAL_DIR/.. || { echo "Failure changing to directory $ACTUAL_DIR/.."; exit 1; } +cd "$ACTUAL_DIR"/.. || { echo "Failure changing to directory $ACTUAL_DIR/.."; exit 1; } ################################################################################ # Build local docker ################################################################################ -printInfo Build docker container kitdm/$REPO_NAME:$TAG_NAME +printInfo Build docker container ghcr.io/kit-data-manager/"$REPO_NAME":"$TAG_NAME" -if ! docker build -t kitdm/$REPO_NAME:$TAG_NAME .; then +if ! docker build -t ghcr.io/kit-data-manager/"$REPO_NAME":"$TAG_NAME" .; then echo . printInfo "ERROR while building docker container!" usage else echo . - printInfo Now you can create and start the container by calling docker "run -d -p8040:8040 --name metastore4docker kitdm/$REPO_NAME:$TAG_NAME" + printInfo Now you can create and start the container by calling docker "run -d -p8040:8040 --name metastore4docker ghcr.io/kit-data-manager/$REPO_NAME:$TAG_NAME" fi diff --git a/docs/documentation.adoc b/docs/documentation.adoc index 505dbfc2..e6fe0300 100644 --- a/docs/documentation.adoc +++ b/docs/documentation.adoc @@ -31,6 +31,21 @@ In between, special characteristics of the calls are explained together with add For technical reasons, all metadata resources shown in the examples contain all fields, e.g. also empty lists or fields with value 'null'. You may ignore most of them as long as they are not needed. Some of them will be assigned by the server, others remain empty or null as long as you don't assign any value to them. All fields mandatory at creation time are explained in the resource creation example. + +=== Building the URL +The URL for accessing the MetaStore REST endpoints is constructed as follows: + +1. Protocol (e.g., http, https) +2. Host name (e.g. localhost, www.example.org) +3. Port (e.g. 8040) +4. Context path (e.g. /metastore) +5. Endpoint (e.g. /api/v1/metadata) + +For example, to list all the schema records in your local MetaStore, you need to +run the following in your browser: http://localhost:8040/metastore/api/v1/schemas + +In former versions (< 1.3.0), no context path was provided by default. + [[XML]] == XML (Schema) [[ChapterMetadataSchemaHandling4Xml]] @@ -63,7 +78,9 @@ At least the following elements are expected to be provided by the user: - schemaId: A unique label for the schema. - type: XML or JSON. For XSD schemas this should be 'XML' -In addition, ACL may be useful to make schema editable by others. (This will be of interest while updating an existing schema) +In addition, ACL may be useful to make schema readable/editable by others. +This will be of interest while accessing/updating an existing schema.(if authorization is enabled) + === Registering a Metadata Schema Document @@ -769,7 +786,7 @@ schema-record4json.json: ---- schema.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -779,7 +796,6 @@ schema.json: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." @@ -852,7 +868,7 @@ Just send an HTTP POST with the updated metadata schema document and/or metadata ---- schema-v2.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -863,13 +879,11 @@ schema-v2.json: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", @@ -900,7 +914,7 @@ Just send an HTTP POST with the updated metadata schema document and/or metadata ---- schema-v3.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -911,20 +925,17 @@ schema-v3.json: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", "description": "Date of object" }, "note": { - "$id": "#/properties/string", "type": "string", "title": "Note", "description": "Additonal information about object" @@ -962,7 +973,7 @@ another-schema-record4json.json: ---- another-schema.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json/example", "type": "object", "title": "Another Json schema for tests", @@ -972,7 +983,6 @@ another-schema.json: ], "properties": { "description": { - "$id": "#/properties/string", "type": "string", "title": "Description", "description": "Any description." diff --git a/gradle.properties b/gradle.properties index bfe52440..ba41272b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,7 +3,7 @@ systemProp.jdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2" //----------------------------- // Properties for build.gradle //----------------------------- -version=1.2.3-SNAPSHOT +version=1.3.1-SNAPSHOT //--------------------------- // Netbeans specific actions @@ -18,3 +18,8 @@ action.custom-4=buildRunnableJar action.custom-4.args=--configure-on-demand -w -x check -Dprofile=minimal clean build action.custom-5=completeTest action.custom-5.args=--configure-on-demand -w -x check clean build jacocoTestReport +action.custom-6=FullCheck +action.custom-6.args=--configure-on-demand -w -x check -x test -Xlint:deprecation _Xlint:unchecked clean build +netbeans.com-github-philippefichet-sonarlint4netbeans.excludedRules=[{"repository":"php","rule":"S3335"},{"repository":"php","rule":"S3336"},{"repository":"php","rule":"S2002"},{"repository":"php","rule":"S2005"},{"repository":"php","rule":"S3333"},{"repository":"php","rule":"S3334"},{"repository":"Web","rule":"ImgWithoutWidthOrHeightCheck"},{"repository":"php","rule":"S1151"},{"repository":"php","rule":"S3332"},{"repository":"php","rule":"S2001"},{"repository":"Web","rule":"UnclosedTagCheck"},{"repository":"php","rule":"S2000"},{"repository":"java","rule":"S2309"},{"repository":"php","rule":"S126"},{"repository":"java","rule":"S4605"},{"repository":"java","rule":"S4604"},{"repository":"java","rule":"S2308"},{"repository":"xml","rule":"S3282"},{"repository":"php","rule":"S2918"},{"repository":"java","rule":"S2301"},{"repository":"java","rule":"S1696"},{"repository":"java","rule":"S1213"},{"repository":"java","rule":"S1698"},{"repository":"java","rule":"S1699"},{"repository":"php","rule":"S1820"},{"repository":"java","rule":"S3750"},{"repository":"java","rule":"S4288"},{"repository":"java","rule":"S1451"},{"repository":"php","rule":"S122"},{"repository":"java","rule":"S1694"},{"repository":"java","rule":"S1695"},{"repository":"php","rule":"S1821"},{"repository":"php","rule":"S5867"},{"repository":"java","rule":"S6211"},{"repository":"java","rule":"S2096"},{"repository":"java","rule":"S6212"},{"repository":"php","rule":"S2011"},{"repository":"Web","rule":"UnifiedExpressionCheck"},{"repository":"Web","rule":"PageWithoutFaviconCheck"},{"repository":"php","rule":"S134"},{"repository":"java","rule":"S1448"},{"repository":"java","rule":"S2658"},{"repository":"java","rule":"S1449"},{"repository":"java","rule":"S3749"},{"repository":"php","rule":"S139"},{"repository":"java","rule":"S5128"},{"repository":"php","rule":"S2007"},{"repository":"php","rule":"S3337"},{"repository":"php","rule":"S3338"},{"repository":"java","rule":"S1200"},{"repository":"Web","rule":"ComplexityCheck"},{"repository":"Web","rule":"RequiredAttributeCheck"},{"repository":"php","rule":"S104"},{"repository":"java","rule":"S3658"},{"repository":"java","rule":"S2208"},{"repository":"java","rule":"S2444"},{"repository":"java","rule":"S2203"},{"repository":"java","rule":"S2325"},{"repository":"java","rule":"S3414"},{"repository":"Web","rule":"S1829"},{"repository":"java","rule":"S4174"},{"repository":"Web","rule":"IllegalTabCheck"},{"repository":"java","rule":"S1106"},{"repository":"java","rule":"S1228"},{"repository":"java","rule":"S1107"},{"repository":"java","rule":"S1108"},{"repository":"java","rule":"S1109"},{"repository":"xml","rule":"S3373"},{"repository":"java","rule":"S1105"},{"repository":"php","rule":"S5856"},{"repository":"java","rule":"S2063"},{"repository":"java","rule":"S3030"},{"repository":"java","rule":"S3032"},{"repository":"Web","rule":"DoubleQuotesCheck"},{"repository":"Web","rule":"MaxLineLengthCheck"},{"repository":"java","rule":"S105"},{"repository":"Web","rule":"LongJavaScriptCheck"},{"repository":"java","rule":"S103"},{"repository":"Web","rule":"WmodeIsWindowCheck"},{"repository":"java","rule":"S104"},{"repository":"java","rule":"S109"},{"repository":"java","rule":"S113"},{"repository":"java","rule":"S4926"},{"repository":"java","rule":"S4248"},{"repository":"java","rule":"S1774"},{"repository":"php","rule":"S1105"},{"repository":"php","rule":"S1106"},{"repository":"php","rule":"S1121"},{"repository":"Web","rule":"InputWithoutLabelCheck"},{"repository":"xml","rule":"S2260"},{"repository":"java","rule":"S2059"},{"repository":"java","rule":"S1641"},{"repository":"java","rule":"S2972"},{"repository":"java","rule":"S2973"},{"repository":"java","rule":"S2974"},{"repository":"java","rule":"S2057"},{"repository":"java","rule":"S6411"},{"repository":"php","rule":"S1117"},{"repository":"java","rule":"S3052"},{"repository":"php","rule":"S1451"},{"repository":"Web","rule":"NonConsecutiveHeadingCheck"},{"repository":"java","rule":"S126"},{"repository":"java","rule":"S923"},{"repository":"java","rule":"S134"},{"repository":"java","rule":"S1315"},{"repository":"java","rule":"S1312"},{"repository":"java","rule":"S1314"},{"repository":"java","rule":"S4266"},{"repository":"java","rule":"S1310"},{"repository":"java","rule":"S2196"},{"repository":"java","rule":"S2197"},{"repository":"java","rule":"S118"},{"repository":"java","rule":"S1309"},{"repository":"java","rule":"S3725"},{"repository":"java","rule":"S121"},{"repository":"java","rule":"S122"},{"repository":"Web","rule":"DynamicJspIncludeCheck"},{"repository":"php","rule":"S1578"},{"repository":"php","rule":"S5935"},{"repository":"Web","rule":"InlineStyleCheck"},{"repository":"java","rule":"S3047"},{"repository":"java","rule":"S1541"},{"repository":"java","rule":"S2260"},{"repository":"java","rule":"S2141"},{"repository":"php","rule":"S1311"},{"repository":"java","rule":"S2384"},{"repository":"php","rule":"S4142"},{"repository":"xml","rule":"S105"},{"repository":"java","rule":"S2701"},{"repository":"xml","rule":"S103"},{"repository":"java","rule":"S2148"},{"repository":"php","rule":"S881"},{"repository":"java","rule":"S2143"},{"repository":"java","rule":"S1176"},{"repository":"java","rule":"S1160"},{"repository":"php","rule":"S1200"},{"repository":"java","rule":"S2250"},{"repository":"java","rule":"S818"},{"repository":"java","rule":"S1162"},{"repository":"java","rule":"S4551"},{"repository":"java","rule":"S2131"},{"repository":"java","rule":"S138"},{"repository":"java","rule":"S139"},{"repository":"Web","rule":"IllegalNamespaceCheck"},{"repository":"php","rule":"S5915"},{"repository":"php","rule":"S1314"},{"repository":"java","rule":"S1166"},{"repository":"php","rule":"S1799"},{"repository":"java","rule":"S5793"},{"repository":"java","rule":"S1194"},{"repository":"java","rule":"S2162"},{"repository":"java","rule":"S2164"},{"repository":"xml","rule":"S3419"},{"repository":"Web","rule":"WhiteSpaceAroundCheck"},{"repository":"java","rule":"S3937"},{"repository":"xml","rule":"S1120"},{"repository":"java","rule":"S1996"},{"repository":"Web","rule":"MultiplePageDirectivesCheck"},{"repository":"xml","rule":"S3420"},{"repository":"java","rule":"S3254"},{"repository":"xml","rule":"S3423"},{"repository":"java","rule":"S2047"},{"repository":"php","rule":"S1541"},{"repository":"Web","rule":"InternationalizationCheck"},{"repository":"java","rule":"S3242"},{"repository":"java","rule":"S6073"},{"repository":"php","rule":"S2070"},{"repository":"java","rule":"S2959"},{"repository":"Web","rule":"LinkToNothingCheck"},{"repository":"Web","rule":"S1436"},{"repository":"java","rule":"S2039"},{"repository":"xml","rule":"S2321"},{"repository":"java","rule":"S1188"},{"repository":"java","rule":"S1067"},{"repository":"java","rule":"S2156"},{"repository":"java","rule":"S3366"},{"repository":"Web","rule":"LinksIdenticalTextsDifferentTargetsCheck"},{"repository":"php","rule":"S2047"},{"repository":"php","rule":"S2046"},{"repository":"java","rule":"S5970"},{"repository":"php","rule":"S2043"},{"repository":"php","rule":"S2042"},{"repository":"php","rule":"S1990"},{"repository":"php","rule":"S2044"},{"repository":"java","rule":"S864"},{"repository":"Web","rule":"MouseEventWithoutKeyboardEquivalentCheck"},{"repository":"java","rule":"S5979"},{"repository":"java","rule":"NoSonar"},{"repository":"java","rule":"S5977"},{"repository":"java","rule":"S5612"},{"repository":"java","rule":"S1258"},{"repository":"java","rule":"S3437"},{"repository":"Web","rule":"FileLengthCheck"},{"repository":"Web","rule":"JspScriptletCheck"},{"repository":"java","rule":"S2221"},{"repository":"java","rule":"S1132"},{"repository":"java","rule":"S3553"},{"repository":"php","rule":"NoSonar"},{"repository":"Web","rule":"IllegalTagLibsCheck"},{"repository":"php","rule":"S2050"},{"repository":"java","rule":"S3306"},{"repository":"java","rule":"S2698"},{"repository":"php","rule":"S1996"},{"repository":"php","rule":"S2964"},{"repository":"java","rule":"S2693"},{"repository":"java","rule":"S1120"},{"repository":"java","rule":"S2694"},{"repository":"java","rule":"S2211"},{"repository":"php","rule":"S1997"},{"repository":"java","rule":"S2333"},{"repository":"java","rule":"S1244"},{"repository":"php","rule":"S5899"},{"repository":"java","rule":"S1151"},{"repository":"Web","rule":"IllegalElementCheck"},{"repository":"java","rule":"S5194"},{"repository":"php","rule":"S2260"},{"repository":"java","rule":"S888"},{"repository":"java","rule":"S1711"},{"repository":"java","rule":"S3578"},{"repository":"php","rule":"S2036"},{"repository":"php","rule":"S2278"},{"repository":"php","rule":"S2277"},{"repository":"php","rule":"S1067"},{"repository":"php","rule":"S2830"},{"repository":"php","rule":"S2038"},{"repository":"php","rule":"S2037"},{"repository":"php","rule":"S5783"},{"repository":"java","rule":"S1939"},{"repository":"java","rule":"S1821"},{"repository":"java","rule":"S1942"},{"repository":"java","rule":"S1943"},{"repository":"java","rule":"S881"},{"repository":"java","rule":"S5867"},{"repository":"java","rule":"S1147"},{"repository":"java","rule":"S1820"},{"repository":"java","rule":"S1941"},{"repository":"java","rule":"S1142"},{"repository":"php","rule":"S2701"},{"repository":"Web","rule":"HeaderCheck"}] +netbeans.com-github-philippefichet-sonarlint4netbeans.rules_2e_parameters_2e_java_2e_S3776_2e_Threshold=15 +netbeans.com-github-philippefichet-sonarlint4netbeans.extraProperties={} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832..033e24c4 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702..9f4197d5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb6..fcb6fca1 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +80,10 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +130,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,6 +197,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/gradlew.bat b/gradlew.bat index 53a6b238..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% diff --git a/metaStoreFramework.sh b/metaStoreFramework.sh deleted file mode 100644 index cf1ef4f6..00000000 --- a/metaStoreFramework.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash -################################################################################ -# Management for metastore framework -# - Managing the following instances -# - elasticsearch -# - rabbitMQ -# - indexing-service -# - metastore2 -# Usage: -# bash metastoreFramework.sh [init|start|stop] -################################################################################ - -################################################################################ -# Define default values for variables -################################################################################ -# no defaults yet! - -################################################################################ -# START DECLARATION FUNCTIONS -################################################################################ - -################################################################################ -function usage { -################################################################################ - echo "Script for managing metastore service." - echo "USAGE:" - echo " $0 [init|start|stop]" - echo " " - echo " init - Initialize/Reset the whole framework" - echo " start - Start stopped framework" - echo " stop - Stop framework" - exit 1 -} - -################################################################################ -function checkParameters { -################################################################################ - # Check no of parameters. - if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters!" - usage - fi -} - -################################################################################ -function initFramework { -################################################################################ -printInfo "Setup Framework" - -echo "Setup configuration directories for metaStore and indexing-Service" -mkdir -p "$ACTUAL_DIR/settings/metastore" -mkdir -p "$ACTUAL_DIR/settings/indexing" - -echo "Setup network for docker..." -docker network create network4datamanager - -echo "Start RabbitMQ server..." -deleteDockerContainer rabbitmq4docker -docker run -d --hostname rabbitmq --net network4datamanager --name rabbitmq4docker -p 5672:5672 -p 15672:15672 rabbitmq:3-management - -echo "Start metaStore2..." -deleteDockerContainer metastore4docker -docker run -d -v "$ACTUAL_DIR/settings/metastore":/spring/metastore2/config --net network4datamanager --name metastore4docker -p8040:8040 kitdm/metastore2:latest - -echo "Start elasticsearch server..." -deleteDockerContainer elasticsearch4metastore -docker run -d --net network4datamanager --name elasticsearch4metastore -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 - -echo "Start Indexing-Service..." -deleteDockerContainer indexing4metastore -docker run -d -v "$ACTUAL_DIR/settings/metastore":/spring/indexing-service/config --net network4datamanager --name indexing4metastore -p 8050:8050 indexing-service:latest - -printInfo "Ready to use metastore" -} - -################################################################################ -function startFramework { -################################################################################ -printInfo "(Re)start metastore and all linked services..." - -echo "Start RabbitMQ server..." -docker start rabbitmq4docker - -echo "Start metastore2..." -docker start metastore4docker - -echo "Start elasticsearch server..." -docker start elasticsearch4metastore - -echo "Start Indexing-Service..." -docker start indexing4docker - -printInfo "Framework started!" -} - -################################################################################ -function stopFramework { -################################################################################ -printInfo "Shutdown metastore and all linked services..." - -echo "Stop Indexing-Service..." -docker stop indexing4docker - -echo "Stop elasticsearch server..." -docker stop elasticsearch4metastore - -echo "Stop metastore2..." -docker stop metastore4docker - -echo "Stop RabbitMQ server..." -docker stop rabbitmq4docker - -printInfo "Framework stopped!" -} - -################################################################################ -function deleteDockerContainer { -################################################################################ -printInfo "Delete docker image '$1'" - -if docker ps | grep -q "$1"; then - echo "Docker container '$1' still running -> Stop docker container" - docker stop $1 -fi - -if docker ps -a | grep -q "$1"; then - echo "Docker container '$1' exists -> Remove docker container" - docker rm $1 -fi -} - -################################################################################ -function printInfo { -################################################################################ -echo "---------------------------------------------------------------------------" -echo "$*" -echo "---------------------------------------------------------------------------" -} - -################################################################################ -# END DECLARATION FUNCTIONS / START OF SCRIPT -################################################################################ - -################################################################################ -# Test for commands used in this script -################################################################################ -testForCommands="type echo grep mkdir docker" - -for command in $testForCommands -do - if ! type $command >> /dev/null; then - echo "Error: command '$command' is not installed!" - exit 1 - fi -done - -################################################################################ -# Determine directory of script. -################################################################################ -ACTUAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -################################################################################ -# Check parameters -################################################################################ -checkParameters "$*" - -################################################################################ -# Manage framework -################################################################################ - -case "$1" in - init) initFramework - ;; - start) startFramework - ;; - stop) stopFramework - ;; - *) usage - ;; -esac diff --git a/restDocu.md b/restDocu.md index 5ac62f5f..ed834923 100644 --- a/restDocu.md +++ b/restDocu.md @@ -26,14 +26,35 @@ block shows the response comming from the server. In between, special characteristics of the calls are explained together with additional, optional arguments or alternative responses. -> **Note** -> -> For technical reasons, all metadata resources shown in the examples -> contain all fields, e.g. also empty lists or fields with value 'null'. -> You may ignore most of them as long as they are not needed. Some of -> them will be assigned by the server, others remain empty or null as -> long as you don’t assign any value to them. All fields mandatory at -> creation time are explained in the resource creation example. +For technical reasons, all metadata resources shown in the examples +contain all fields, e.g. also empty lists or fields with value 'null'. +You may ignore most of them as long as they are not needed. Some of them +will be assigned by the server, others remain empty or null as long as +you don’t assign any value to them. All fields mandatory at creation +time are explained in the resource creation example. + +Building the URL +---------------- + +The URL for accessing the MetaStore REST endpoints is constructed as +follows: + +1. Protocol (e.g., http, https) + +2. Host name (e.g. localhost, www.example.org) + +3. Port (e.g. 8040) + +4. Context path (e.g. /metastore) + +5. Endpoint (e.g. /api/v1/metadata) + +For example, to list all the schema records in your local MetaStore, you +need to run the following in your browser: + + +In former versions (< 1.3.0), no context path was provided by +default. XML (Schema) ============ @@ -68,8 +89,9 @@ At least the following elements are expected to be provided by the user: - type: XML or JSON. For XSD schemas this should be *XML* -In addition, ACL may be useful to make schema editable by others. (This -will be of interest while updating an existing schema) +In addition, ACL may be useful to make schema readable/editable by +others. This will be of interest while accessing/updating an existing +schema.(if authorization is enabled) Registering a Metadata Schema Document -------------------------------------- @@ -98,7 +120,7 @@ providing mandatory fields mentioned above: - $ curl 'http://localhost:8040/api/v1/schemas' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'schema=@schema.xsd;type=application/xml' \ -F 'record=@schema-record.json;type=application/json' @@ -107,7 +129,7 @@ You can see, that most of the sent metadata schema record is empty. Only schemaId, mimeType and type are provided by the user. HTTP-wise the call looks as follows: - POST /api/v1/schemas HTTP/1.1 + POST /metastore/api/v1/schemas HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -143,24 +165,24 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/schemas/my_first_xsd?version=1 - ETag: "701164283" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1 + ETag: "343002905" Content-Type: application/json - Content-Length: 457 + Content-Length: 467 { "schemaId" : "my_first_xsd", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:54.418Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.695Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "schemaHash" : "sha1:08b262fe74604d6d5d001ed03718408e52bae9aa", "doNotSync" : true } @@ -177,20 +199,18 @@ DELETE, in order to avoid conflicts. For obtaining one metadata schema record you have to provide the value of the field 'schemaId'. -> **Note** -> -> As 'Accept' field you have to provide -> 'application/vnd.datamanager.schema-record+json' otherwise you will -> get the metadata schema instead. +As 'Accept' field you have to provide +'application/vnd.datamanager.schema-record+json' otherwise you will get +the metadata schema instead. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X GET \ -H 'Accept: application/vnd.datamanager.schema-record+json' In the actual HTTP request just access the path of the resource using the base path and the 'schemaid'. Be aware that you also have to provide the 'Accept' field. - GET /api/v1/schemas/my_first_xsd HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Accept: application/vnd.datamanager.schema-record+json Host: localhost:8040 @@ -198,23 +218,23 @@ As a result, you receive the metadata schema record send before and again the corresponding ETag in the HTTP response header. HTTP/1.1 200 OK - ETag: "701164283" + ETag: "343002905" Content-Type: application/vnd.datamanager.schema-record+json - Content-Length: 457 + Content-Length: 467 { "schemaId" : "my_first_xsd", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:54.418Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.695Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "schemaHash" : "sha1:08b262fe74604d6d5d001ed03718408e52bae9aa", "doNotSync" : true } @@ -225,12 +245,12 @@ For obtaining accessible metadata schemas you also have to provide the 'schemaId'. For accessing schema document you don’t have to provide the 'Accept' header. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X GET In the actual HTTP request there is nothing special. You just access the path of the resource using the base path and the 'schemaId'. - GET /api/v1/schemas/my_first_xsd HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Host: localhost:8040 As a result, you receive the XSD schema send before. @@ -261,11 +281,9 @@ As a result, you receive the XSD schema send before. ### Updating a Metadata Schema Document (add mandatory 'date' field) -> **Note** -> -> Updating a metadata schema document will not break old metadata -> documents. As every update results in a new version 'old' metadata -> schema documents are still available. +Updating a metadata schema document will not break old metadata +documents. As every update results in a new version 'old' metadata +schema documents are still available. For updating an existing metadata schema (record) a valid ETag is needed. The actual ETag is available via the HTTP GET call of the @@ -288,16 +306,16 @@ updated metadata schema document and/or metadata schema record. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "701164283"' \ + -H 'If-Match: "343002905"' \ -F 'schema=@schema-v2.xsd;type=application/xml' HTTP-wise the call looks as follows: - PUT /api/v1/schemas/my_first_xsd HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "701164283" + If-Match: "343002905" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm @@ -325,24 +343,24 @@ As a result, you receive the updated schema record and in the HTTP response header the new location URL and the ETag. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_xsd?version=2 - ETag: "700011989" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2 + ETag: "1480363722" Content-Type: application/json - Content-Length: 457 + Content-Length: 467 { "schemaId" : "my_first_xsd", "schemaVersion" : 2, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:55.274Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.835Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=2", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2", "schemaHash" : "sha1:c227b2bf612264da33fe5a695b5450101ce9d766", "doNotSync" : true } @@ -370,16 +388,16 @@ document and/or metadata schema record. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "700011989"' \ + -H 'If-Match: "1480363722"' \ -F 'schema=@schema-v3.xsd;type=application/xml' HTTP-wise the call looks as follows: - PUT /api/v1/schemas/my_first_xsd HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "700011989" + If-Match: "1480363722" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm @@ -408,24 +426,24 @@ As a result, you receive the updated schema record and in the HTTP response header the new location URL and the ETag. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_xsd?version=3 - ETag: "2120896070" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3 + ETag: "2106056635" Content-Type: application/json - Content-Length: 457 + Content-Length: 466 { "schemaId" : "my_first_xsd", "schemaVersion" : 3, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:55.447Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.87Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "schemaHash" : "sha1:1baea3a07d95faea70707fcf46d114315613b970", "doNotSync" : true } @@ -459,14 +477,14 @@ providing mandatory fields mentioned above: - $ curl 'http://localhost:8040/api/v1/schemas' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'schema=@another-schema.xsd;type=application/xml' \ -F 'record=@another-schema-record.json;type=application/json' HTTP-wise the call looks as follows: - POST /api/v1/schemas HTTP/1.1 + POST /metastore/api/v1/schemas HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -502,24 +520,24 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/schemas/another_xsd?version=1 - ETag: "497145878" + Location: http://localhost:8040/metastore/api/v1/schemas/another_xsd?version=1 + ETag: "-786384830" Content-Type: application/json - Content-Length: 455 + Content-Length: 465 { "schemaId" : "another_xsd", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:55Z", - "lastUpdate" : "2021-08-13T10:00:55.528Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.893Z", "acl" : [ { "id" : 2, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/another_xsd?version=1", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/another_xsd?version=1", "schemaHash" : "sha1:c834a580e7b74c66650a8b00640e4f16cfab7bac", "doNotSync" : true } @@ -530,33 +548,33 @@ Now there are two schemaIds registered in the metadata schema registry. Obtaining all accessible metadata schema records. - $ curl 'http://localhost:8040/api/v1/schemas' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X GET Same for HTTP request: - GET /api/v1/schemas HTTP/1.1 + GET /metastore/api/v1/schemas HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata schema records. HTTP/1.1 200 OK - Content-Range: 0-19/2 + Content-Range: 0-9/2 Content-Type: application/json - Content-Length: 918 + Content-Length: 937 [ { "schemaId" : "another_xsd", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:55Z", - "lastUpdate" : "2021-08-13T10:00:55.528Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.893Z", "acl" : [ { "id" : 2, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/another_xsd?version=1", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/another_xsd?version=1", "schemaHash" : "sha1:c834a580e7b74c66650a8b00640e4f16cfab7bac", "doNotSync" : true }, { @@ -564,28 +582,24 @@ As a result, you receive a list of metadata schema records. "schemaVersion" : 3, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:55.447Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.87Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "schemaHash" : "sha1:1baea3a07d95faea70707fcf46d114315613b970", "doNotSync" : true } ] -> **Note** -> -> Only the current version of each schemaId is listed. +Only the current version of each schemaId is listed. -> **Note** -> -> The header contains the field 'Content-Range" which displays delivered -> indices and the maximum number of available schema records. If there -> are more than 20 schemas registered you have to provide page and/or -> size as additional query parameters. +The header contains the field 'Content-Range" which displays delivered +indices and the maximum number of available schema records. If there are +more than 20 schemas registered you have to provide page and/or size as +additional query parameters. - page: Number of the page you want to get **(starting with page 0)** @@ -593,7 +607,7 @@ As a result, you receive a list of metadata schema records. The modified HTTP request with pagination looks like follows: - GET /api/v1/schemas?page=0&size=20 HTTP/1.1 + GET /metastore/api/v1/schemas?page=0&size=20 HTTP/1.1 Host: localhost:8040 ### Getting a List of all Schema Records for a Specific SchemaId @@ -601,34 +615,34 @@ The modified HTTP request with pagination looks like follows: If you want to obtain all versions of a specific schema you may add the schemaId as a filter parameter. This may look like this: - $ curl 'http://localhost:8040/api/v1/schemas?schemaId=my_first_xsd' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas?schemaId=my_first_xsd' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas?schemaId=my_first_xsd HTTP/1.1 + GET /metastore/api/v1/schemas?schemaId=my_first_xsd HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata schema records in descending order. (current version first) HTTP/1.1 200 OK - Content-Range: 0-19/3 + Content-Range: 0-9/3 Content-Type: application/json - Content-Length: 1379 + Content-Length: 1408 [ { "schemaId" : "my_first_xsd", "schemaVersion" : 3, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:55.447Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.87Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "schemaHash" : "sha1:1baea3a07d95faea70707fcf46d114315613b970", "doNotSync" : true }, { @@ -636,14 +650,14 @@ order. (current version first) "schemaVersion" : 2, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:55.274Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.835Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=2", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2", "schemaHash" : "sha1:c227b2bf612264da33fe5a695b5450101ce9d766", "doNotSync" : true }, { @@ -651,14 +665,14 @@ order. (current version first) "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:54.418Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:24.695Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "schemaHash" : "sha1:08b262fe74604d6d5d001ed03718408e52bae9aa", "doNotSync" : true } ] @@ -668,11 +682,11 @@ order. (current version first) To get the current version of the metadata schema document just send an HTTP GET with the linked 'schemaId': - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas/my_first_xsd HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Host: localhost:8040 As a result, you receive the XSD schema document sent before. @@ -711,11 +725,11 @@ To get a specific version of the metadata schema document just send an HTTP GET with the linked 'schemaId' and the version number you are looking for as query parameter: - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd?version=1' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas/my_first_xsd?version=1 HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_xsd?version=1 HTTP/1.1 Host: localhost:8040 As a result, you receive the initial XSD schema document (version 1). @@ -760,22 +774,17 @@ the schemaVersion to validate given document. On a first step validation with the old schema will be done: - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd/validate?version=1' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd/validate?version=1' -i -X POST \ -H 'Content-Type: multipart/form-data' \ - -F 'document=@metadata-v3.xml;type=application/xml' \ - -F 'version=1' + -F 'document=@metadata-v3.xml;type=application/xml' Same for the HTTP request. The schemaVersion number is set by a query parameter. - POST /api/v1/schemas/my_first_xsd/validate?version=1 HTTP/1.1 + POST /metastore/api/v1/schemas/my_first_xsd/validate?version=1 HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 - --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - Content-Disposition: form-data; name=version - - 1 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v3.xml Content-Type: application/xml @@ -793,19 +802,29 @@ some information about the error. (Unfortunately not documented here due to technical reasons.) HTTP/1.1 422 Unprocessable Entity + Content-Type: application/problem+json + Content-Length: 314 + + { + "type" : "about:blank", + "title" : "Unprocessable Entity", + "status" : 422, + "detail" : "Validation error: cvc-complex-type.2.4.d: Invalid content was found starting with element 'example:date'. No child element is expected at this point.", + "instance" : "/metastore/api/v1/schemas/my_first_xsd/validate" + } The document holds a mandatory and an optional field introduced in the second and third version of schema. Let’s try to validate with third version of schema. Only version number will be different. (if no query parameter is available the current version will be selected) - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd/validate' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd/validate' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'document=@metadata-v3.xml;type=application/xml' Same for the HTTP request. - POST /api/v1/schemas/my_first_xsd/validate HTTP/1.1 + POST /metastore/api/v1/schemas/my_first_xsd/validate HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -851,41 +870,41 @@ example we introduce a user called 'admin' and give him all rights. ] } - $ curl 'http://localhost:8040/api/v1/schemas/my_first_xsd' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_xsd' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "2120896070"' \ + -H 'If-Match: "2106056635"' \ -F 'record=@schema-record-v4.json;type=application/json' Same for the HTTP request. - PUT /api/v1/schemas/my_first_xsd HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_xsd HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "2120896070" + If-Match: "2106056635" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=schema-record-v4.json Content-Type: application/json - {"schemaId":"my_first_xsd","pid":null,"schemaVersion":3,"label":null,"definition":null,"comment":null,"mimeType":"application/xml","type":"XML","createdAt":"2021-08-13T10:00:54Z","lastUpdate":"2021-08-13T10:00:55.447Z","acl":[{"id":1,"sid":"SELF","permission":"ADMINISTRATE"},{"id":null,"sid":"admin","permission":"ADMINISTRATE"}],"schemaDocumentUri":"http://localhost:8040/api/v1/schemas/my_first_xsd?version=3","schemaHash":"sha1:1baea3a07d95faea70707fcf46d114315613b970","doNotSync":true} + {"schemaId":"my_first_xsd","pid":null,"schemaVersion":3,"label":null,"definition":null,"comment":null,"mimeType":"application/xml","type":"XML","createdAt":"2023-07-12T12:45:24Z","lastUpdate":"2023-07-12T12:45:24.87Z","acl":[{"id":1,"sid":"SELF","permission":"ADMINISTRATE"},{"id":null,"sid":"admin","permission":"ADMINISTRATE"}],"schemaDocumentUri":"http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3","schemaHash":"sha1:1baea3a07d95faea70707fcf46d114315613b970","doNotSync":true} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- As a result, you receive 200 as HTTP status, the updated metadata schema record and the updated ETag and location in the HTTP response header. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_xsd?version=3 - ETag: "811149141" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3 + ETag: "-229236150" Content-Type: application/json - Content-Length: 532 + Content-Length: 543 { "schemaId" : "my_first_xsd", "schemaVersion" : 3, "mimeType" : "application/xml", "type" : "XML", - "createdAt" : "2021-08-13T10:00:54Z", - "lastUpdate" : "2021-08-13T10:00:56.01Z", + "createdAt" : "2023-07-12T12:45:24Z", + "lastUpdate" : "2023-07-12T12:45:25.147Z", "acl" : [ { "id" : 1, "sid" : "SELF", @@ -895,7 +914,7 @@ record and the updated ETag and location in the HTTP response header. "sid" : "admin", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "schemaHash" : "sha1:1baea3a07d95faea70707fcf46d114315613b970", "doNotSync" : true } @@ -949,10 +968,8 @@ At least the following elements are expected to be provided by the user: In addition, ACL may be useful to make metadata editable by others. (This will be of interest while updating an existing metadata) -> **Note** -> -> If linked schema is identified by its schemaId the INTERNAL type has -> to be used. +If linked schema is identified by its schemaId the INTERNAL type has to +be used. ### Register/Ingest a Metadata Record with Metadata Document @@ -981,7 +998,7 @@ and its metadata only providing mandatory fields mentioned above: The schemaId used while registering metadata schema has to be used to link the metadata with the approbriate metadata schema. - $ curl 'http://localhost:8040/api/v1/metadata' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'record=@metadata-record.json;type=application/json' \ -F 'document=@metadata.xml;type=application/xml' @@ -990,7 +1007,7 @@ You can see, that most of the sent metadata schema record is empty. Only schemaId and relatedResource are provided by the user. HTTP-wise the call looks as follows: - POST /api/v1/metadata HTTP/1.1 + POST /metastore/api/v1/metadata HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -1016,25 +1033,21 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1 - ETag: "-334720106" + Location: http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1 + ETag: "-769713607" Content-Type: application/json - Content-Length: 806 + Content-Length: 709 { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.363Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.2Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -1044,7 +1057,7 @@ the user and will look that way: "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1", "documentHash" : "sha1:ac92891f6377919446143e0a8639f12715397228" } @@ -1061,12 +1074,12 @@ avoid conflicts. For accessing the metadata the location URL provided before may be used. The URL is compiled by the id of the metadata and its version. - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1' -i -X GET \ -H 'Accept: application/xml' HTTP-wise the call looks as follows: - GET /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1 HTTP/1.1 + GET /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1 HTTP/1.1 Accept: application/xml Host: localhost:8040 @@ -1094,12 +1107,12 @@ The only difference is the content type. It has to be set to "application/vnd.datamanager.metadata-record+json". Then the command line looks like this: - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1' -i -X GET \ -H 'Accept: application/vnd.datamanager.metadata-record+json' HTTP-wise the call looks as follows: - GET /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1 HTTP/1.1 + GET /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1 HTTP/1.1 Accept: application/vnd.datamanager.metadata-record+json Host: localhost:8040 @@ -1107,24 +1120,20 @@ The linked metadata will be returned. The result is sent back to the user and will look that way: HTTP/1.1 200 OK - ETag: "-334720106" + ETag: "-769713607" Content-Type: application/vnd.datamanager.metadata-record+json - Content-Length: 806 + Content-Length: 709 { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.363Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.2Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -1134,7 +1143,7 @@ user and will look that way: "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1", "documentHash" : "sha1:ac92891f6377919446143e0a8639f12715397228" } @@ -1171,25 +1180,25 @@ the ETag is needed: 2018-07-02 - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "-334720106"' \ + -H 'If-Match: "-769713607"' \ -F 'record=@metadata-record-v2.json;type=application/json' \ -F 'document=@metadata-v2.xml;type=application/xml' You can see, that only the ACL entry for "guest" was added. All other properties are still the same. HTTP-wise the call looks as follows: - PUT /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b HTTP/1.1 + PUT /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64 HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "-334720106" + If-Match: "-769713607" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=metadata-record-v2.json Content-Type: application/json - {"id":"ed533130-9e74-43fb-82db-e2464fbb0b1b","pid":{"id":null,"identifier":"ed533130-9e74-43fb-82db-e2464fbb0b1b","identifierType":"INTERNAL"},"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"INTERNAL"},"createdAt":"2021-08-13T10:00:56Z","lastUpdate":"2021-08-13T10:00:56.363Z","schema":{"id":null,"identifier":"my_first_xsd","identifierType":"INTERNAL"},"schemaVersion":2,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1","documentHash":"sha1:ac92891f6377919446143e0a8639f12715397228"} + {"id":"7a07631d-d999-42bb-bd64-32d45838fe64","pid":null,"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"URL"},"createdAt":"2023-07-12T12:45:25Z","lastUpdate":"2023-07-12T12:45:25.2Z","schema":{"id":null,"identifier":"my_first_xsd","identifierType":"INTERNAL"},"schemaVersion":2,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1","documentHash":"sha1:ac92891f6377919446143e0a8639f12715397228"} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v2.xml Content-Type: application/xml @@ -1202,39 +1211,35 @@ properties are still the same. HTTP-wise the call looks as follows: --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- You will get the new metadata record with the additional ACL entry. -Version number was incremented by one, 'lastUpdate' and *recordVersion* -are also modified by the server. +Version number of record was incremented by one and 'lastUpdate' was +also modified by the server. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2 - ETag: "831113235" + Location: http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2 + ETag: "-561646093" Content-Type: application/json - Content-Length: 806 + Content-Length: 711 { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.761Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.348Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2", "documentHash" : "sha1:e13a87884df391a611fb6257ea53883811d9451a" } @@ -1247,46 +1252,42 @@ Repeat the last step and update to the current version. As mentioned before the ETag is needed. As the ETag has changed in the meanwhile you first have to get the new ETag. - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2' -i -X GET \ -H 'Accept: application/vnd.datamanager.metadata-record+json' HTTP-wise the call looks as follows: - GET /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2 HTTP/1.1 + GET /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2 HTTP/1.1 Accept: application/vnd.datamanager.metadata-record+json Host: localhost:8040 You will get the new metadata record with the new ETag. HTTP/1.1 200 OK - ETag: "831113235" + ETag: "-561646093" Content-Type: application/vnd.datamanager.metadata-record+json - Content-Length: 806 + Content-Length: 711 { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.761Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.348Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2", "documentHash" : "sha1:e13a87884df391a611fb6257ea53883811d9451a" } @@ -1314,24 +1315,24 @@ Etag. since version 3 notes are allowed - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "831113235"' \ + -H 'If-Match: "-561646093"' \ -F 'record=@metadata-record-v3.json;type=application/json' \ -F 'document=@metadata-v3.xml;type=application/xml' HTTP-wise the call looks as follows: - PUT /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b HTTP/1.1 + PUT /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64 HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "831113235" + If-Match: "-561646093" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=metadata-record-v3.json Content-Type: application/json - {"id":"ed533130-9e74-43fb-82db-e2464fbb0b1b","pid":{"id":null,"identifier":"ed533130-9e74-43fb-82db-e2464fbb0b1b","identifierType":"INTERNAL"},"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"INTERNAL"},"createdAt":"2021-08-13T10:00:56Z","lastUpdate":"2021-08-13T10:00:56.363Z","schema":{"id":null,"identifier":"my_first_xsd","identifierType":"INTERNAL"},"schemaVersion":3,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1","documentHash":"sha1:ac92891f6377919446143e0a8639f12715397228"} + {"id":"7a07631d-d999-42bb-bd64-32d45838fe64","pid":null,"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"URL"},"createdAt":"2023-07-12T12:45:25Z","lastUpdate":"2023-07-12T12:45:25.2Z","schema":{"id":null,"identifier":"my_first_xsd","identifierType":"INTERNAL"},"schemaVersion":3,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1","documentHash":"sha1:ac92891f6377919446143e0a8639f12715397228"} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v3.xml Content-Type: application/xml @@ -1347,46 +1348,42 @@ HTTP-wise the call looks as follows: You will get the new metadata record. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3 - ETag: "-1343323824" + Location: http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3 + ETag: "1386118128" Content-Type: application/json - Content-Length: 806 + Content-Length: 711 { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.952Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.428Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3", "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" } Now you can access the updated metadata via the URI in the HTTP response header. - $ curl 'http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3 HTTP/1.1 + GET /metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3 HTTP/1.1 Host: localhost:8040 You will get the updated metadata. @@ -1421,12 +1418,10 @@ are set via query parameters. The following filters are allowed: - until -> **Note** -> -> The header contains the field 'Content-Range" which displays delivered -> indices and the maximum number of available schema records. If there -> are more than 20 metadata records registered you have to provide page -> and/or size as additional query parameters. +The header contains the field 'Content-Range" which displays delivered +indices and the maximum number of available schema records. If there are +more than 20 metadata records registered you have to provide page and/or +size as additional query parameters. - page: Number of the page you want to get **(starting with page 0)** @@ -1437,85 +1432,73 @@ are set via query parameters. The following filters are allowed: If you want to obtain all versions of a specific resource you may add 'id' as a filter parameter. This may look like this: - $ curl 'http://localhost:8040/api/v1/metadata?id=ed533130-9e74-43fb-82db-e2464fbb0b1b' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?id=7a07631d-d999-42bb-bd64-32d45838fe64' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?id=ed533130-9e74-43fb-82db-e2464fbb0b1b HTTP/1.1 + GET /metastore/api/v1/metadata?id=7a07631d-d999-42bb-bd64-32d45838fe64 HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata records in descending order. (current version first) HTTP/1.1 200 OK - Content-Range: 0-19/3 + Content-Range: 0-9/3 Content-Type: application/json - Content-Length: 2426 + Content-Length: 2139 [ { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.952Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.428Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3", "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" }, { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.761Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.348Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=2", - "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=2", + "documentHash" : "sha1:e13a87884df391a611fb6257ea53883811d9451a" }, { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.363Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.2Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -1525,8 +1508,8 @@ As a result, you receive a list of metadata records in descending order. "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=1", - "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=1", + "documentHash" : "sha1:ac92891f6377919446143e0a8639f12715397228" } ] #### Find by resourceId @@ -1537,44 +1520,40 @@ MetaStore may hold multiple metadata documents per resource. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId HTTP/1.1 + GET /metastore/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId HTTP/1.1 Host: localhost:8040 You will get the current version of the metadata record(s). HTTP/1.1 200 OK - Content-Range: 0-19/1 + Content-Range: 0-9/1 Content-Type: application/json - Content-Length: 810 + Content-Length: 715 [ { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.952Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.428Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3", "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" } ] @@ -1584,45 +1563,41 @@ If you want to find all metadata records updated after a specific date. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?from=2021-08-13T08%3A00%3A57.109937Z' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A25.517721088Z' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?from=2021-08-13T08%3A00%3A57.109937Z HTTP/1.1 + GET /metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A25.517721088Z HTTP/1.1 Host: localhost:8040 You will get the current version metadata records updated ln the last 2 hours. HTTP/1.1 200 OK - Content-Range: 0-19/1 + Content-Range: 0-9/1 Content-Type: application/json - Content-Length: 810 + Content-Length: 715 [ { - "id" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "pid" : { - "identifier" : "ed533130-9e74-43fb-82db-e2464fbb0b1b", - "identifierType" : "INTERNAL" - }, + "id" : "7a07631d-d999-42bb-bd64-32d45838fe64", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:00:56Z", - "lastUpdate" : "2021-08-13T10:00:56.952Z", + "createdAt" : "2023-07-12T12:45:25Z", + "lastUpdate" : "2023-07-12T12:45:25.428Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_xsd?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_xsd?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/ed533130-9e74-43fb-82db-e2464fbb0b1b?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/7a07631d-d999-42bb-bd64-32d45838fe64?version=3", "documentHash" : "sha1:55547a0ad07445cfbc11a76484da3b21d23ceb82" } ] @@ -1633,18 +1608,18 @@ range. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?from=2021-08-13T08%3A00%3A57.109937Z&until=2021-08-13T09%3A00%3A57.109926Z' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A25.517721088Z&until=2023-07-12T11%3A45%3A25.517718361Z' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?from=2021-08-13T08%3A00%3A57.109937Z&until=2021-08-13T09%3A00%3A57.109926Z HTTP/1.1 + GET /metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A25.517721088Z&until=2023-07-12T11%3A45%3A25.517718361Z HTTP/1.1 Host: localhost:8040 You will get an empty array as no metadata record exists in the given range: HTTP/1.1 200 OK - Content-Range: 0-19/0 + Content-Range: 0-9/0 Content-Type: application/json Content-Length: 3 @@ -1703,7 +1678,7 @@ providing mandatory fields mentioned above: schema.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -1713,7 +1688,6 @@ providing mandatory fields mentioned above: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." @@ -1722,7 +1696,7 @@ providing mandatory fields mentioned above: "additionalProperties": false } - $ curl 'http://localhost:8040/api/v1/schemas' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'schema=@schema.json;type=application/json' \ -F 'record=@schema-record4json.json;type=application/json' @@ -1731,7 +1705,7 @@ You can see, that most of the sent metadata schema record is empty. Only schemaId, mimeType and type are provided by the user. HTTP-wise the call looks as follows: - POST /api/v1/schemas HTTP/1.1 + POST /metastore/api/v1/schemas HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -1740,7 +1714,7 @@ looks as follows: Content-Type: application/json { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -1750,7 +1724,6 @@ looks as follows: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." @@ -1772,25 +1745,25 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/schemas/my_first_json?version=1 - ETag: "-1424777959" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1 + ETag: "1946363153" Content-Type: application/json - Content-Length: 461 + Content-Length: 471 { "schemaId" : "my_first_json", "schemaVersion" : 1, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:01.926Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:20.442Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", - "schemaHash" : "sha1:98741f0d6115474ab69375be3bc9cd5b305ce200", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", + "schemaHash" : "sha1:a312efb2ad92b1c8ce8f6556c40e2b185c6682c4", "doNotSync" : true } @@ -1806,20 +1779,18 @@ DELETE, in order to avoid conflicts. For obtaining one metadata schema record you have to provide the value of the field 'schemaId'. -> **Note** -> -> As 'Accept' field you have to provide -> 'application/vnd.datamanager.schema-record+json' otherwise you will -> get the metadata schema instead. +As 'Accept' field you have to provide +'application/vnd.datamanager.schema-record+json' otherwise you will get +the metadata schema instead. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X GET \ -H 'Accept: application/vnd.datamanager.schema-record+json' In the actual HTTP request just access the path of the resource using the base path and the 'schemaid'. Be aware that you also have to provide the 'Accept' field. - GET /api/v1/schemas/my_first_json HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_json HTTP/1.1 Accept: application/vnd.datamanager.schema-record+json Host: localhost:8040 @@ -1827,24 +1798,24 @@ As a result, you receive the metadata schema record send before and again the corresponding ETag in the HTTP response header. HTTP/1.1 200 OK - ETag: "-1424777959" + ETag: "1946363153" Content-Type: application/vnd.datamanager.schema-record+json - Content-Length: 461 + Content-Length: 471 { "schemaId" : "my_first_json", "schemaVersion" : 1, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:01.926Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:20.442Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", - "schemaHash" : "sha1:98741f0d6115474ab69375be3bc9cd5b305ce200", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", + "schemaHash" : "sha1:a312efb2ad92b1c8ce8f6556c40e2b185c6682c4", "doNotSync" : true } @@ -1854,23 +1825,23 @@ For obtaining accessible metadata schemas you also have to provide the 'schemaId'. For accessing schema document you don’t have to provide the 'Accept' header. - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X GET In the actual HTTP request there is nothing special. You just access the path of the resource using the base path and the 'schemaId'. - GET /api/v1/schemas/my_first_json HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_json HTTP/1.1 Host: localhost:8040 As a result, you receive the XSD schema send before. HTTP/1.1 200 OK Content-Type: application/json - Content-Length: 425 + Content-Length: 388 Accept-Ranges: bytes { - "$schema" : "http://json-schema.org/draft/2019-09/schema#", + "$schema" : "https://json-schema.org/draft/2020-12/schema", "$id" : "http://www.example.org/schema/json", "type" : "object", "title" : "Json schema for tests", @@ -1878,7 +1849,6 @@ As a result, you receive the XSD schema send before. "required" : [ "title" ], "properties" : { "title" : { - "$id" : "#/properties/string", "type" : "string", "title" : "Title", "description" : "Title of object." @@ -1889,11 +1859,9 @@ As a result, you receive the XSD schema send before. ### Updating a Metadata Schema Document (add mandatory 'date' field) -> **Note** -> -> Updating a metadata schema document will not break old metadata -> documents. As every update results in a new version 'old' metadata -> schema documents are still available. +Updating a metadata schema document will not break old metadata +documents. As every update results in a new version 'old' metadata +schema documents are still available. For updating an existing metadata schema (record) a valid ETag is needed. The actual ETag is available via the HTTP GET call of the @@ -1902,7 +1870,7 @@ updated metadata schema document and/or metadata schema record. schema-v2.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -1913,13 +1881,11 @@ updated metadata schema document and/or metadata schema record. ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", @@ -1929,16 +1895,16 @@ updated metadata schema document and/or metadata schema record. "additionalProperties": false } - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "-1424777959"' \ + -H 'If-Match: "1946363153"' \ -F 'schema=@schema-v2.json;type=application/json' HTTP-wise the call looks as follows: - PUT /api/v1/schemas/my_first_json HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_json HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "-1424777959" + If-Match: "1946363153" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm @@ -1946,7 +1912,7 @@ HTTP-wise the call looks as follows: Content-Type: application/json { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -1957,13 +1923,11 @@ HTTP-wise the call looks as follows: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", @@ -1978,25 +1942,25 @@ As a result, you receive the updated schema record and in the HTTP response header the new location URL and the ETag. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_json?version=2 - ETag: "-2004027021" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2 + ETag: "-1107116181" Content-Type: application/json - Content-Length: 461 + Content-Length: 471 { "schemaId" : "my_first_json", "schemaVersion" : 2, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:02.336Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:21.276Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=2", - "schemaHash" : "sha1:68c72ab169770015f9b68645d0a50ac33a98f46c", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2", + "schemaHash" : "sha1:6d65072f514d887d9fa69997d3aa067524c1c085", "doNotSync" : true } @@ -2008,7 +1972,7 @@ document and/or metadata schema record. schema-v3.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -2019,20 +1983,17 @@ document and/or metadata schema record. ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", "description": "Date of object" }, "note": { - "$id": "#/properties/string", "type": "string", "title": "Note", "description": "Additonal information about object" @@ -2041,16 +2002,16 @@ document and/or metadata schema record. "additionalProperties": false } - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "-2004027021"' \ + -H 'If-Match: "-1107116181"' \ -F 'schema=@schema-v3.json;type=application/json' HTTP-wise the call looks as follows: - PUT /api/v1/schemas/my_first_json HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_json HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "-2004027021" + If-Match: "-1107116181" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm @@ -2058,7 +2019,7 @@ HTTP-wise the call looks as follows: Content-Type: application/json { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json", "type": "object", "title": "Json schema for tests", @@ -2069,20 +2030,17 @@ HTTP-wise the call looks as follows: ], "properties": { "title": { - "$id": "#/properties/string", "type": "string", "title": "Title", "description": "Title of object." }, "date": { - "$id": "#/properties/string", "type": "string", "format": "date", "title": "Date", "description": "Date of object" }, "note": { - "$id": "#/properties/string", "type": "string", "title": "Note", "description": "Additonal information about object" @@ -2096,25 +2054,25 @@ As a result, you receive the updated schema record and in the HTTP response header the new location URL and the ETag. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_json?version=3 - ETag: "1569865828" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3 + ETag: "-1097218276" Content-Type: application/json - Content-Length: 461 + Content-Length: 471 { "schemaId" : "my_first_json", "schemaVersion" : 3, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:02.699Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:21.512Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", - "schemaHash" : "sha1:150dc302a01dbd35f360d4f09540fce859bfcd32", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", + "schemaHash" : "sha1:16221eb6fd0177135b873acd78da1a221a8b621d", "doNotSync" : true } @@ -2135,7 +2093,7 @@ providing mandatory fields mentioned above: another-schema.json: { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json/example", "type": "object", "title": "Another Json schema for tests", @@ -2145,7 +2103,6 @@ providing mandatory fields mentioned above: ], "properties": { "description": { - "$id": "#/properties/string", "type": "string", "title": "Description", "description": "Any description." @@ -2154,14 +2111,14 @@ providing mandatory fields mentioned above: "additionalProperties": false } - $ curl 'http://localhost:8040/api/v1/schemas' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'schema=@another-schema.json;type=application/xml' \ - -F 'record=@another-schema-record4json.json;type=application/json' + -F 'record=@another-schema-record.json;type=application/json' HTTP-wise the call looks as follows: - POST /api/v1/schemas HTTP/1.1 + POST /metastore/api/v1/schemas HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -2170,7 +2127,7 @@ HTTP-wise the call looks as follows: Content-Type: application/xml { - "$schema": "http://json-schema.org/draft/2019-09/schema#", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "http://www.example.org/schema/json/example", "type": "object", "title": "Another Json schema for tests", @@ -2180,7 +2137,6 @@ HTTP-wise the call looks as follows: ], "properties": { "description": { - "$id": "#/properties/string", "type": "string", "title": "Description", "description": "Any description." @@ -2202,25 +2158,25 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/schemas/another_json?version=1 - ETag: "-1476067843" + Location: http://localhost:8040/metastore/api/v1/schemas/another_json?version=1 + ETag: "1336866933" Content-Type: application/json - Content-Length: 458 + Content-Length: 468 { "schemaId" : "another_json", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:02Z", - "lastUpdate" : "2021-08-13T10:01:02.973Z", + "createdAt" : "2023-07-12T12:45:21Z", + "lastUpdate" : "2023-07-12T12:45:21.727Z", "acl" : [ { "id" : 2, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/another_json?version=1", - "schemaHash" : "sha1:ea94d4b621fe2d66a69c3f1a3b5c91665b1714cd", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/another_json?version=1", + "schemaHash" : "sha1:a4bd131a0cbd65000fa5db6ca71d45379348adb4", "doNotSync" : true } @@ -2230,62 +2186,58 @@ Now there are two schemaIds registered in the metadata schema registry. Obtaining all accessible metadata schema records. - $ curl 'http://localhost:8040/api/v1/schemas' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas' -i -X GET Same for HTTP request: - GET /api/v1/schemas HTTP/1.1 + GET /metastore/api/v1/schemas HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata schema records. HTTP/1.1 200 OK - Content-Range: 0-19/2 + Content-Range: 0-9/2 Content-Type: application/json - Content-Length: 925 + Content-Length: 945 [ { "schemaId" : "another_json", "schemaVersion" : 1, "mimeType" : "application/xml", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:02Z", - "lastUpdate" : "2021-08-13T10:01:02.973Z", + "createdAt" : "2023-07-12T12:45:21Z", + "lastUpdate" : "2023-07-12T12:45:21.727Z", "acl" : [ { "id" : 2, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/another_json?version=1", - "schemaHash" : "sha1:ea94d4b621fe2d66a69c3f1a3b5c91665b1714cd", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/another_json?version=1", + "schemaHash" : "sha1:a4bd131a0cbd65000fa5db6ca71d45379348adb4", "doNotSync" : true }, { "schemaId" : "my_first_json", "schemaVersion" : 3, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:02.699Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:21.512Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", - "schemaHash" : "sha1:150dc302a01dbd35f360d4f09540fce859bfcd32", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", + "schemaHash" : "sha1:16221eb6fd0177135b873acd78da1a221a8b621d", "doNotSync" : true } ] -> **Note** -> -> Only the current version of each schemaId is listed. +Only the current version of each schemaId is listed. -> **Note** -> -> The header contains the field 'Content-Range" which displays delivered -> indices and the maximum number of available schema records. If there -> are more than 20 schemas registered you have to provide page and/or -> size as additional query parameters. +The header contains the field 'Content-Range" which displays delivered +indices and the maximum number of available schema records. If there are +more than 20 schemas registered you have to provide page and/or size as +additional query parameters. - page: Number of the page you want to get **(starting with page 0)** @@ -2293,7 +2245,7 @@ As a result, you receive a list of metadata schema records. The modified HTTP request with pagination looks like follows: - GET /api/v1/schemas?page=0&size=20 HTTP/1.1 + GET /metastore/api/v1/schemas?page=0&size=20 HTTP/1.1 Host: localhost:8040 ### Getting a List of all Schema Records for a Specific SchemaId @@ -2301,65 +2253,65 @@ The modified HTTP request with pagination looks like follows: If you want to obtain all versions of a specific schema you may add the schemaId as a filter parameter. This may look like this: - $ curl 'http://localhost:8040/api/v1/schemas?schemaId=my_first_json' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas?schemaId=my_first_json' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas?schemaId=my_first_json HTTP/1.1 + GET /metastore/api/v1/schemas?schemaId=my_first_json HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata schema records in descending order. (current version first) HTTP/1.1 200 OK - Content-Range: 0-19/3 + Content-Range: 0-9/3 Content-Type: application/json - Content-Length: 1391 + Content-Length: 1421 [ { "schemaId" : "my_first_json", "schemaVersion" : 3, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:02.699Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:21.512Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", - "schemaHash" : "sha1:150dc302a01dbd35f360d4f09540fce859bfcd32", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", + "schemaHash" : "sha1:16221eb6fd0177135b873acd78da1a221a8b621d", "doNotSync" : true }, { "schemaId" : "my_first_json", "schemaVersion" : 2, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:02.336Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:21.276Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=2", - "schemaHash" : "sha1:68c72ab169770015f9b68645d0a50ac33a98f46c", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2", + "schemaHash" : "sha1:6d65072f514d887d9fa69997d3aa067524c1c085", "doNotSync" : true }, { "schemaId" : "my_first_json", "schemaVersion" : 1, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:01.926Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:20.442Z", "acl" : [ { "id" : 1, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", - "schemaHash" : "sha1:98741f0d6115474ab69375be3bc9cd5b305ce200", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", + "schemaHash" : "sha1:a312efb2ad92b1c8ce8f6556c40e2b185c6682c4", "doNotSync" : true } ] @@ -2368,22 +2320,22 @@ order. (current version first) To get the current version of the metadata schema document just send an HTTP GET with the linked 'schemaId': - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas/my_first_json HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_json HTTP/1.1 Host: localhost:8040 As a result, you receive the XSD schema document sent before. HTTP/1.1 200 OK Content-Type: application/json - Content-Length: 772 + Content-Length: 661 Accept-Ranges: bytes { - "$schema" : "http://json-schema.org/draft/2019-09/schema#", + "$schema" : "https://json-schema.org/draft/2020-12/schema", "$id" : "http://www.example.org/schema/json", "type" : "object", "title" : "Json schema for tests", @@ -2391,20 +2343,17 @@ As a result, you receive the XSD schema document sent before. "required" : [ "title", "date" ], "properties" : { "title" : { - "$id" : "#/properties/string", "type" : "string", "title" : "Title", "description" : "Title of object." }, "date" : { - "$id" : "#/properties/string", "type" : "string", "format" : "date", "title" : "Date", "description" : "Date of object" }, "note" : { - "$id" : "#/properties/string", "type" : "string", "title" : "Note", "description" : "Additonal information about object" @@ -2419,22 +2368,22 @@ To get a specific version of the metadata schema document just send an HTTP GET with the linked 'schemaId' and the version number you are looking for as query parameter: - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json?version=1' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/schemas/my_first_json?version=1 HTTP/1.1 + GET /metastore/api/v1/schemas/my_first_json?version=1 HTTP/1.1 Host: localhost:8040 As a result, you receive the initial XSD schema document (version 1). HTTP/1.1 200 OK Content-Type: application/json - Content-Length: 425 + Content-Length: 388 Accept-Ranges: bytes { - "$schema" : "http://json-schema.org/draft/2019-09/schema#", + "$schema" : "https://json-schema.org/draft/2020-12/schema", "$id" : "http://www.example.org/schema/json", "type" : "object", "title" : "Json schema for tests", @@ -2442,7 +2391,6 @@ As a result, you receive the initial XSD schema document (version 1). "required" : [ "title" ], "properties" : { "title" : { - "$id" : "#/properties/string", "type" : "string", "title" : "Title", "description" : "Title of object." @@ -2466,22 +2414,17 @@ the schemaVersion to validate given document. On a first step validation with the old schema will be done: - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json/validate?version=1' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json/validate?version=1' -i -X POST \ -H 'Content-Type: multipart/form-data' \ - -F 'document=@metadata-v3.json;type=application/json' \ - -F 'version=1' + -F 'document=@metadata-v3.json;type=application/json' Same for the HTTP request. The schemaVersion number is set by a query parameter. - POST /api/v1/schemas/my_first_json/validate?version=1 HTTP/1.1 + POST /metastore/api/v1/schemas/my_first_json/validate?version=1 HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 - --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - Content-Disposition: form-data; name=version - - 1 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v3.json Content-Type: application/json @@ -2498,19 +2441,29 @@ some information about the error. (Unfortunately not documented here due to technical reasons.) HTTP/1.1 422 Unprocessable Entity + Content-Type: application/problem+json + Content-Length: 376 + + { + "type" : "about:blank", + "title" : "Unprocessable Entity", + "status" : 422, + "detail" : "400 BAD_REQUEST \"Error validating json!\n$.date ist nicht im Schema definiert und das Schema verbietet additionalProperties\n$.note ist nicht im Schema definiert und das Schema verbietet additionalProperties\"", + "instance" : "/metastore/api/v1/schemas/my_first_json/validate" + } The document holds a mandatory and an optional field introduced in the second and third version of schema. Let’s try to validate with third version of schema. Only version number will be different. (if no query parameter is available the current version will be selected) - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json/validate' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json/validate' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'document=@metadata-v3.json;type=application/json' Same for the HTTP request. - POST /api/v1/schemas/my_first_json/validate HTTP/1.1 + POST /metastore/api/v1/schemas/my_first_json/validate HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -2552,41 +2505,41 @@ example we introduce a user called 'admin' and give him all rights. } ] } - $ curl 'http://localhost:8040/api/v1/schemas/my_first_json' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/schemas/my_first_json' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "1569865828"' \ + -H 'If-Match: "-1097218276"' \ -F 'record=@schema-record4json-v4.json;type=application/json' Same for the HTTP request. - PUT /api/v1/schemas/my_first_json HTTP/1.1 + PUT /metastore/api/v1/schemas/my_first_json HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "1569865828" + If-Match: "-1097218276" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=schema-record4json-v4.json Content-Type: application/json - {"schemaId":"my_first_json","pid":null,"schemaVersion":3,"label":null,"definition":null,"comment":null,"mimeType":"application/json","type":"JSON","createdAt":"2021-08-13T10:01:01Z","lastUpdate":"2021-08-13T10:01:02.699Z","acl":[{"id":1,"sid":"SELF","permission":"ADMINISTRATE"},{"id":null,"sid":"admin","permission":"ADMINISTRATE"}],"schemaDocumentUri":"http://localhost:8040/api/v1/schemas/my_first_json?version=3","schemaHash":"sha1:150dc302a01dbd35f360d4f09540fce859bfcd32","doNotSync":true} + {"schemaId":"my_first_json","pid":null,"schemaVersion":3,"label":null,"definition":null,"comment":null,"mimeType":"application/json","type":"JSON","createdAt":"2023-07-12T12:45:20Z","lastUpdate":"2023-07-12T12:45:21.512Z","acl":[{"id":1,"sid":"SELF","permission":"ADMINISTRATE"},{"id":null,"sid":"admin","permission":"ADMINISTRATE"}],"schemaDocumentUri":"http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3","schemaHash":"sha1:16221eb6fd0177135b873acd78da1a221a8b621d","doNotSync":true} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- As a result, you receive 200 as HTTP status, the updated metadata schema record and the updated ETag and location in the HTTP response header. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/schemas/my_first_json?version=3 - ETag: "-1660325133" + Location: http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3 + ETag: "-2006054549" Content-Type: application/json - Content-Length: 537 + Content-Length: 547 { "schemaId" : "my_first_json", "schemaVersion" : 3, "mimeType" : "application/json", "type" : "JSON", - "createdAt" : "2021-08-13T10:01:01Z", - "lastUpdate" : "2021-08-13T10:01:03.296Z", + "createdAt" : "2023-07-12T12:45:20Z", + "lastUpdate" : "2023-07-12T12:45:22.258Z", "acl" : [ { "id" : 1, "sid" : "SELF", @@ -2596,8 +2549,8 @@ record and the updated ETag and location in the HTTP response header. "sid" : "admin", "permission" : "ADMINISTRATE" } ], - "schemaDocumentUri" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", - "schemaHash" : "sha1:150dc302a01dbd35f360d4f09540fce859bfcd32", + "schemaDocumentUri" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", + "schemaHash" : "sha1:16221eb6fd0177135b873acd78da1a221a8b621d", "doNotSync" : true } @@ -2647,8 +2600,8 @@ At least the following elements are expected to be provided by the user: - relatedResource: The link to the resource. -metadataDocuIn addition, ACL may be useful to make metadata editable by -others. (This will be of interest while updating an existing metadata) +In addition, ACL may be useful to make metadata editable by others. +(This will be of interest while updating an existing metadata) ### Register/Ingest a Metadata Record with Metadata Document @@ -2676,7 +2629,7 @@ and its metadata only providing mandatory fields mentioned above: The schemaId used while registering metadata schema has to be used to link the metadata with the approbriate metadata schema. - $ curl 'http://localhost:8040/api/v1/metadata' -i -X POST \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata' -i -X POST \ -H 'Content-Type: multipart/form-data' \ -F 'record=@metadata-record4json.json;type=application/json' \ -F 'document=@metadata.json;type=application/json' @@ -2685,7 +2638,7 @@ You can see, that most of the sent metadata schema record is empty. Only schemaId and relatedResource are provided by the user. HTTP-wise the call looks as follows: - POST /api/v1/metadata HTTP/1.1 + POST /metastore/api/v1/metadata HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Host: localhost:8040 @@ -2710,25 +2663,21 @@ possible and persisting the created resource, the result is sent back to the user and will look that way: HTTP/1.1 201 Created - Location: http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1 - ETag: "812848657" + Location: http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1 + ETag: "1695569906" Content-Type: application/json - Content-Length: 807 + Content-Length: 712 { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.406Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.364Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -2738,7 +2687,7 @@ the user and will look that way: "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1", "documentHash" : "sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a" } @@ -2755,12 +2704,12 @@ avoid conflicts. For accessing the metadata the location URL provided before may be used. The URL is compiled by the id of the metadata and its version. - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1' -i -X GET \ -H 'Accept: application/json' HTTP-wise the call looks as follows: - GET /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1 HTTP/1.1 + GET /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1 HTTP/1.1 Accept: application/json Host: localhost:8040 @@ -2785,12 +2734,12 @@ The only difference is the content type. It has to be set to "application/vnd.datamanager.metadata-record+json". Then the command line looks like this: - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1' -i -X GET \ -H 'Accept: application/vnd.datamanager.metadata-record+json' HTTP-wise the call looks as follows: - GET /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1 HTTP/1.1 + GET /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1 HTTP/1.1 Accept: application/vnd.datamanager.metadata-record+json Host: localhost:8040 @@ -2798,24 +2747,20 @@ The linked metadata will be returned. The result is sent back to the user and will look that way: HTTP/1.1 200 OK - ETag: "812848657" + ETag: "1695569906" Content-Type: application/vnd.datamanager.metadata-record+json - Content-Length: 807 + Content-Length: 712 { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.406Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.364Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -2825,7 +2770,7 @@ user and will look that way: "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1", "documentHash" : "sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a" } @@ -2860,30 +2805,25 @@ mentioned before the ETag is needed: "date": "2018-07-02" } - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "812848657"' \ + -H 'If-Match: "1695569906"' \ -F 'record=@metadata-record4json-v2.json;type=application/json' \ - -F 'document=@metadata-v2.json;type=application/xml' \ - -F 'version=1' + -F 'document=@metadata-v2.json;type=application/xml' You can see, that only the ACL entry for "guest" was added. All other properties are still the same. HTTP-wise the call looks as follows: - PUT /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1 HTTP/1.1 + PUT /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1 HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "812848657" + If-Match: "1695569906" Host: localhost:8040 - --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - Content-Disposition: form-data; name=version - - 1 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=metadata-record4json-v2.json Content-Type: application/json - {"id":"e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2","pid":{"id":null,"identifier":"e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2","identifierType":"INTERNAL"},"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"INTERNAL"},"createdAt":"2021-08-13T10:01:03Z","lastUpdate":"2021-08-13T10:01:03.406Z","schema":{"id":null,"identifier":"my_first_json","identifierType":"INTERNAL"},"schemaVersion":2,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1","documentHash":"sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a"} + {"id":"290d4094-8b58-4f16-ba85-d0cc88a1f81e","pid":null,"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"URL"},"createdAt":"2023-07-12T12:45:22Z","lastUpdate":"2023-07-12T12:45:22.364Z","schema":{"id":null,"identifier":"my_first_json","identifierType":"INTERNAL"},"schemaVersion":2,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1","documentHash":"sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a"} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v2.json Content-Type: application/xml @@ -2895,39 +2835,35 @@ properties are still the same. HTTP-wise the call looks as follows: --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm-- You will get the new metadata record with the additional ACL entry. -Version number was incremented by one, 'lastUpdate' and *recordVersion* -are also modified by the server. +Version number of record was incremented by one and 'lastUpdate' was +also modified by the server. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2 - ETag: "-398051314" + Location: http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2 + ETag: "834161808" Content-Type: application/json - Content-Length: 807 + Content-Length: 712 { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.694Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.565Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2", "documentHash" : "sha1:1844c8057b673ae260fcc6b6ba146529b2b52771" } @@ -2940,46 +2876,42 @@ Repeat the last step and update to the current version. As mentioned before the ETag is needed. As the ETag has changed in the meanwhile you first have to get the new ETag. - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2' -i -X GET \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2' -i -X GET \ -H 'Accept: application/vnd.datamanager.metadata-record+json' HTTP-wise the call looks as follows: - GET /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2 HTTP/1.1 + GET /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2 HTTP/1.1 Accept: application/vnd.datamanager.metadata-record+json Host: localhost:8040 You will get the new metadata record with the new ETag. HTTP/1.1 200 OK - ETag: "-398051314" + ETag: "834161808" Content-Type: application/vnd.datamanager.metadata-record+json - Content-Length: 807 + Content-Length: 712 { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.694Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.565Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2", "documentHash" : "sha1:1844c8057b673ae260fcc6b6ba146529b2b52771" } @@ -3006,24 +2938,24 @@ Etag. "note": "since version 3 notes are allowed" } - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2' -i -X PUT \ + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e' -i -X PUT \ -H 'Content-Type: multipart/form-data' \ - -H 'If-Match: "-398051314"' \ + -H 'If-Match: "834161808"' \ -F 'record=@metadata-record4json-v3.json;type=application/json' \ -F 'document=@metadata-v3.json;type=application/xml' HTTP-wise the call looks as follows: - PUT /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2 HTTP/1.1 + PUT /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e HTTP/1.1 Content-Type: multipart/form-data; boundary=6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm - If-Match: "-398051314" + If-Match: "834161808" Host: localhost:8040 --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=record; filename=metadata-record4json-v3.json Content-Type: application/json - {"id":"e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2","pid":{"id":null,"identifier":"e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2","identifierType":"INTERNAL"},"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"INTERNAL"},"createdAt":"2021-08-13T10:01:03Z","lastUpdate":"2021-08-13T10:01:03.406Z","schema":{"id":null,"identifier":"my_first_json","identifierType":"INTERNAL"},"schemaVersion":3,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1","documentHash":"sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a"} + {"id":"290d4094-8b58-4f16-ba85-d0cc88a1f81e","pid":null,"relatedResource":{"id":null,"identifier":"https://repo/anyResourceId","identifierType":"URL"},"createdAt":"2023-07-12T12:45:22Z","lastUpdate":"2023-07-12T12:45:22.364Z","schema":{"id":null,"identifier":"my_first_json","identifierType":"INTERNAL"},"schemaVersion":3,"recordVersion":1,"acl":[{"id":4,"sid":"SELF","permission":"ADMINISTRATE"}],"metadataDocumentUri":"http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1","documentHash":"sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a"} --6o2knFse3p53ty9dmcQvWAIx1zInP11uCfbm Content-Disposition: form-data; name=document; filename=metadata-v3.json Content-Type: application/xml @@ -3038,46 +2970,42 @@ HTTP-wise the call looks as follows: You will get the new metadata record. HTTP/1.1 200 OK - Location: http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3 - ETag: "-765011637" + Location: http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3 + ETag: "2104122253" Content-Type: application/json - Content-Length: 807 + Content-Length: 712 { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.853Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.657Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3", "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" } Now you can access the updated metadata via the URI in the HTTP response header. - $ curl 'http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3 HTTP/1.1 + GET /metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3 HTTP/1.1 Host: localhost:8040 You will get the updated metadata. @@ -3107,12 +3035,10 @@ are set via query parameters. The following filters are allowed: - until -> **Note** -> -> The header contains the field 'Content-Range" which displays delivered -> indices and the maximum number of available schema records. If there -> are more than 20 metadata records registered you have to provide page -> and/or size as additional query parameters. +The header contains the field 'Content-Range" which displays delivered +indices and the maximum number of available schema records. If there are +more than 20 metadata records registered you have to provide page and/or +size as additional query parameters. - page: Number of the page you want to get **(starting with page 0)** @@ -3123,85 +3049,73 @@ are set via query parameters. The following filters are allowed: If you want to obtain all versions of a specific resource you may add 'id' as a filter parameter. This may look like this: - $ curl 'http://localhost:8040/api/v1/metadata?id=e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?id=290d4094-8b58-4f16-ba85-d0cc88a1f81e' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?id=e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2 HTTP/1.1 + GET /metastore/api/v1/metadata?id=290d4094-8b58-4f16-ba85-d0cc88a1f81e HTTP/1.1 Host: localhost:8040 As a result, you receive a list of metadata records in descending order. (current version first) HTTP/1.1 200 OK - Content-Range: 0-19/3 + Content-Range: 0-9/3 Content-Type: application/json - Content-Length: 2429 + Content-Length: 2144 [ { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.853Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.657Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3", "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" }, { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.694Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.565Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=2", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=2", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 2, "recordVersion" : 2, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=2", - "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=2", + "documentHash" : "sha1:1844c8057b673ae260fcc6b6ba146529b2b52771" }, { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.406Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.364Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=1", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=1", "identifierType" : "URL" }, "schemaVersion" : 1, @@ -3211,8 +3125,8 @@ As a result, you receive a list of metadata records in descending order. "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=1", - "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=1", + "documentHash" : "sha1:97ac2fb17cd40aac07a55444dc161d615c70af8a" } ] #### Find by resourceId @@ -3223,44 +3137,40 @@ MetaStore may hold multiple metadata documents per resource. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId HTTP/1.1 + GET /metastore/api/v1/metadata?resoureId=https%3A%2F%2Frepo%2FanyResourceId HTTP/1.1 Host: localhost:8040 You will get the current version metadata record. HTTP/1.1 200 OK - Content-Range: 0-19/1 + Content-Range: 0-9/1 Content-Type: application/json - Content-Length: 811 + Content-Length: 716 [ { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.853Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.657Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3", "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" } ] @@ -3270,45 +3180,41 @@ If you want to find all metadata records updated after a specific date. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?from=2021-08-13T08%3A01%3A04.004539Z' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A22.771448462Z' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?from=2021-08-13T08%3A01%3A04.004539Z HTTP/1.1 + GET /metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A22.771448462Z HTTP/1.1 Host: localhost:8040 You will get the current version metadata records updated ln the last 2 hours. HTTP/1.1 200 OK - Content-Range: 0-19/1 + Content-Range: 0-9/1 Content-Type: application/json - Content-Length: 811 + Content-Length: 716 [ { - "id" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "pid" : { - "identifier" : "e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2", - "identifierType" : "INTERNAL" - }, + "id" : "290d4094-8b58-4f16-ba85-d0cc88a1f81e", "relatedResource" : { "identifier" : "https://repo/anyResourceId", - "identifierType" : "INTERNAL" + "identifierType" : "URL" }, - "createdAt" : "2021-08-13T10:01:03Z", - "lastUpdate" : "2021-08-13T10:01:03.853Z", + "createdAt" : "2023-07-12T12:45:22Z", + "lastUpdate" : "2023-07-12T12:45:22.657Z", "schema" : { - "identifier" : "http://localhost:8040/api/v1/schemas/my_first_json?version=3", + "identifier" : "http://localhost:8040/metastore/api/v1/schemas/my_first_json?version=3", "identifierType" : "URL" }, - "schemaVersion" : 1, + "schemaVersion" : 3, "recordVersion" : 3, "acl" : [ { "id" : 4, "sid" : "SELF", "permission" : "ADMINISTRATE" } ], - "metadataDocumentUri" : "http://localhost:8040/api/v1/metadata/e37a71ed-4a9d-4754-ad47-82d7bdfdf7c2?version=3", + "metadataDocumentUri" : "http://localhost:8040/metastore/api/v1/metadata/290d4094-8b58-4f16-ba85-d0cc88a1f81e?version=3", "documentHash" : "sha1:737762db675032231ac3cb872fccd32a83ac24d1" } ] @@ -3319,18 +3225,18 @@ range. Command line: - $ curl 'http://localhost:8040/api/v1/metadata?from=2021-08-13T08%3A01%3A04.004539Z&until=2021-08-13T09%3A01%3A04.004520Z' -i -X GET + $ curl 'http://localhost:8040/metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A22.771448462Z&until=2023-07-12T11%3A45%3A22.771444065Z' -i -X GET HTTP-wise the call looks as follows: - GET /api/v1/metadata?from=2021-08-13T08%3A01%3A04.004539Z&until=2021-08-13T09%3A01%3A04.004520Z HTTP/1.1 + GET /metastore/api/v1/metadata?from=2023-07-12T10%3A45%3A22.771448462Z&until=2023-07-12T11%3A45%3A22.771444065Z HTTP/1.1 Host: localhost:8040 You will get an empty array as no metadata record exists in the given range: HTTP/1.1 200 OK - Content-Range: 0-19/0 + Content-Range: 0-9/0 Content-Type: application/json Content-Length: 3 diff --git a/settings/application-default.properties b/settings/application-default.properties index f8f1ffc9..4bcf4fd0 100644 --- a/settings/application-default.properties +++ b/settings/application-default.properties @@ -5,17 +5,61 @@ ############################################################################### server.port: 8040 +############################################################################### +# Context Path +############################################################################### +server.servlet.context-path=/metastore + ############################################################################### # Proxy (enable this line if you want to use the service behind a proxy.) ############################################################################### #server.forward-headers-strategy=framework ############################################################################### +# Landing Page (Schema & Metadata Documents) +# Redirect to internal or external landing page. +# The string may contain two placeholders for substitution: +# - $(id) - Identifier of the digital object (mandatory) +# - $(version) - Version of the digital object (optional) +# +# e.g.: https://www.example.org/landingpage?id=$(id)&version=$(version) +# +# For frontend-collection use metastore-landing-page.html?pid=$(id) +# e.g. https://HOSTNAME/metastore-landing-page.html?pid=$(id)&version=$(version) +# +# Defaults: +# - /schema-landing-page?schemaId=$(id)&version=$(version) (schema documents) +# - /metadata-landing-page?id=$(id)&version=$(version) (metadata documents) +############################################################################### +metastore.schema.landingpage:/schema-landing-page?schemaId=$(id)&version=$(version) +metastore.metadata.landingpage:/metadata-landing-page?id=$(id)&version=$(version) + +################################################################################ # Setup paths for schema and metadata ############################################################################### metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -46,7 +90,7 @@ repo.plugin.doip.defaultToken:REPLACE_BY_YOUR_TOKEN # Logging settings ############################################################################### logging.level.root: ERROR -logging.level.edu.kit: WARN +logging.level.edu.kit.datamanager: WARN ############################################################################### # KIT DM settings @@ -66,12 +110,14 @@ repo.schedule.rate:1000 repo.messaging.enabled: false repo.messaging.hostname:localhost repo.messaging.port:5672 +repo.messaging.username:guest +repo.messaging.password:guest repo.messaging.sender.exchange: metastore_events repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. @@ -79,7 +125,7 @@ repo.messaging.receiver.routingKeys: metadata.# repo.search.enabled = false repo.search.url = http://localhost:9200 -############################################################################## +############################################################################### # Database ############################################################################### spring.datasource.driver-class-name: org.h2.Driver @@ -144,3 +190,9 @@ spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfig #keycloak.realm = myrealm #keycloak.auth-server-url = http://localhost:8080/auth #keycloak.resource = keycloak-angular + +############################################################################### +# Due to bug in spring cloud gateway +# https://github.com/spring-cloud/spring-cloud-gateway/issues/3154 +############################################################################### +spring.cloud.gateway.proxy.sensitive=content-length diff --git a/settings/application-docker.properties b/settings/application-docker.properties index f7f7a999..51ed76db 100644 --- a/settings/application-docker.properties +++ b/settings/application-docker.properties @@ -5,12 +5,61 @@ ############################################################################### server.port: 8040 +############################################################################### +# Context Path +############################################################################### +server.servlet.context-path=/metastore + +############################################################################### +# Proxy (enable this line if you want to use the service behind a proxy.) +############################################################################### +#server.forward-headers-strategy=framework + +############################################################################### +# Landing Page (Schema & Metadata Documents) +# Redirect to internal or external landing page. +# The string may contain two placeholders for substitution: +# - $(id) - Identifier of the digital object (mandatory) +# - $(version) - Version of the digital object (optional) +# +# e.g.: https://www.example.org/landingpage?id=$(id)&version=$(version) +# +# For frontend-collection use metastore-landing-page.html?pid=$(id) +# e.g. https://HOSTNAME/metastore-landing-page.html?pid=$(id)&version=$(version) +# +# Defaults: +# - /schema-landing-page?schemaId=$(id)&version=$(version) (schema documents) +# - /metadata-landing-page?id=$(id)&version=$(version) (metadata documents) +############################################################################### +metastore.schema.landingpage:/schema-landing-page?schemaId=$(id)&version=$(version) +metastore.metadata.landingpage:http://localhost/metastore-landing-page.html?pid=$(id)&version=$(version) + ############################################################################### # Setup paths for schema and metadata ############################################################################### metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -57,15 +106,17 @@ repo.auth.jwtSecret:vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxs # Messaging - RabbitMQ ############################################################################### repo.schedule.rate:1000 -repo.messaging.enabled: true +repo.messaging.enabled: false repo.messaging.hostname:rabbitmq.docker repo.messaging.port:5672 +repo.messaging.username:guest +repo.messaging.password:guest repo.messaging.sender.exchange: metastore_events repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. @@ -138,3 +189,9 @@ spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfig #keycloak.realm = myrealm #keycloak.auth-server-url = http://localhost:8080/auth #keycloak.resource = keycloak-angular + +############################################################################### +# Due to bug in spring cloud gateway +# https://github.com/spring-cloud/spring-cloud-gateway/issues/3154 +############################################################################### +spring.cloud.gateway.proxy.sensitive=content-length diff --git a/settings/application-postgres.properties b/settings/application-postgres.properties index 9d7409ca..210d22f6 100644 --- a/settings/application-postgres.properties +++ b/settings/application-postgres.properties @@ -5,17 +5,61 @@ ############################################################################### server.port: 8040 +############################################################################### +# Context Path +############################################################################### +server.servlet.context-path=/metastore + ############################################################################### # Proxy (enable this line if you want to use the service behind a proxy.) ############################################################################### #server.forward-headers-strategy=framework +############################################################################### +# Landing Page (Schema & Metadata Documents) +# Redirect to internal or external landing page. +# The string may contain two placeholders for substitution: +# - $(id) - Identifier of the digital object (mandatory) +# - $(version) - Version of the digital object (optional) +# +# e.g.: https://www.example.org/landingpage?id=$(id)&version=$(version) +# +# For frontend-collection use metastore-landing-page.html?pid=$(id) +# e.g. https://HOSTNAME/metastore-landing-page.html?pid=$(id)&version=$(version) +# +# Defaults: +# - /schema-landing-page?schemaId=$(id)&version=$(version) (schema documents) +# - /metadata-landing-page?id=$(id)&version=$(version) (metadata documents) +############################################################################### +metastore.schema.landingpage:/schema-landing-page?schemaId=$(id)&version=$(version) +metastore.metadata.landingpage:/metadata-landing-page?id=$(id)&version=$(version) + ############################################################################### # Setup paths for schema and metadata ############################################################################### metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -65,12 +109,14 @@ repo.schedule.rate:1000 repo.messaging.enabled: false repo.messaging.hostname:localhost repo.messaging.port:5672 +repo.messaging.username:guest +repo.messaging.password:guest repo.messaging.sender.exchange: metastore_events repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. @@ -149,3 +195,9 @@ spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfig #keycloak.realm = myrealm #keycloak.auth-server-url = http://localhost:8080/auth #keycloak.resource = keycloak-angular + +############################################################################### +# Due to bug in spring cloud gateway +# https://github.com/spring-cloud/spring-cloud-gateway/issues/3154 +############################################################################### +spring.cloud.gateway.proxy.sensitive=content-length diff --git a/setupMetastoreFrameworkWithDocker.md b/setupMetastoreFrameworkWithDocker.md index c39cb948..1757613d 100644 --- a/setupMetastoreFrameworkWithDocker.md +++ b/setupMetastoreFrameworkWithDocker.md @@ -1,101 +1,24 @@ -# How to Setup a complete Framework with Docker +# How to Setup a complete Framework with Docker Compose -## Step 1 - Create network for docker: -``` -docker network create network4datamanager -``` - -## Step 2 - Start RabbitMQ server: -``` -docker run -d --hostname rabbitmq --net network4datamanager --name rabbitmq4docker rabbitmq:3-management -``` - -## Step 3 - Start metastore2: -``` -docker run -d -p8040:8040 --net network4datamanager --name metastore4docker kitdm/metastore2:latest -``` - -## Step 4 - Start elasticsearch server: -``` -docker run -d --net network4datamanager --name elasticsearch4metastore -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 -``` - -## Step 5 - Start Indexing-Service: -``` -docker run -d --net network4datamanager --name indexing4metastore -p 8050:8050 indexing-service:latest -``` - -## Step 6 - Register schema (for metastore2) -``` -``` - -## Step 7 - Register mapping (for elasticsearch): -``` -``` - -## Step 8 - You're ready to ingest your metadata to metastore: -``` -``` - -## Step 9 - All metadata is now available via elasticsearch. +# Create Docker Framework + Build and start framework: ``` +docker compose up ``` # Stop Docker Framework - - -## Step 1 - Stop Indexing-Service: -``` -docker stop indexing4docker -``` - -## Step 2 - Stop elasticsearch server: -``` -docker stop elasticsearch4metastore + Stop framework: ``` - -## Step 3 - Stop metastore2: -``` -docker stop metastore4docker -``` - -## Step 4 - Stop RabbitMQ server: -``` -docker stop rabbitmq4docker +docker compose stop ``` - -# Start Docker Framework -## Step 1 - Start RabbitMQ server: +# Restart Docker Framework + Restart framework: ``` -docker start rabbitmq4docker +docker compose start ``` +# Reset Framework -## Step 2 - Start metastore2: -``` -docker start metastore4docker -``` -## Step 3 - Start elasticsearch server: ``` -docker start elasticsearch4metastore +docker compose down ``` -## Step 4 - Start Indexing-Service: -``` -docker start indexing4docker -``` +For more information type: 'docker compose --help' \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/metastore2/Application.java b/src/main/java/edu/kit/datamanager/metastore2/Application.java index 5ef3dfd9..39e7408a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/Application.java +++ b/src/main/java/edu/kit/datamanager/metastore2/Application.java @@ -34,7 +34,6 @@ import edu.kit.datamanager.repo.configuration.DateBasedStorageProperties; import edu.kit.datamanager.repo.configuration.IdBasedStorageProperties; import edu.kit.datamanager.repo.configuration.StorageServiceProperties; -import edu.kit.datamanager.repo.dao.IDataResourceDao; import edu.kit.datamanager.repo.domain.ContentInformation; import edu.kit.datamanager.repo.domain.DataResource; import edu.kit.datamanager.repo.service.IContentInformationService; @@ -45,7 +44,6 @@ import edu.kit.datamanager.repo.service.impl.ContentInformationService; import edu.kit.datamanager.repo.service.impl.DataResourceAuditService; import edu.kit.datamanager.repo.service.impl.DataResourceService; -import edu.kit.datamanager.repo.service.impl.IdBasedStorageService; import edu.kit.datamanager.security.filter.KeycloakJwtProperties; import edu.kit.datamanager.security.filter.KeycloakTokenFilter; import edu.kit.datamanager.security.filter.KeycloakTokenValidator; @@ -56,6 +54,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.javers.core.Javers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +66,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -77,6 +75,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; /** + * Main class starting spring boot service of MetaStore. */ @SpringBootApplication @EnableScheduling @@ -85,25 +84,22 @@ @ComponentScan({"edu.kit.datamanager"}) public class Application { + private static final String DEFAULT = "simple"; + private static final String DEFAULT_VERSIONING = DEFAULT; + private static final String DEFAULT_STORAGE = DEFAULT; + private static final String LIST_ITEM = ".... '{}'"; + private static final Logger LOG = LoggerFactory.getLogger(Application.class); @Autowired private Javers javers; - /*@Autowired - private IDataResourceService schemaResourceService; - @Autowired - private IContentInformationService schemaInformationService;*/ @Autowired private ApplicationEventPublisher eventPublisher; @Autowired private ApplicationProperties applicationProperties; @Autowired - private IRepoVersioningService[] versioningServices; - @Autowired - private IRepoStorageService[] storageServices; - - private MetastoreConfiguration metastoreProperties; + private List versioningServices; @Autowired - private IDataResourceDao dataResourceDao; + private ListstorageServices; @Autowired private ISchemaRecordDao schemaRecordDao; @Autowired @@ -113,14 +109,8 @@ public class Application { @Autowired private IMetadataFormatDao metadataFormatDao; @Autowired - private OaiPmhConfiguration oaiPmhConfiguration; - @Autowired - private IValidator[] validators; + private List validators; - /*@Autowired - private IDataResourceService dataResourceService; - @Autowired - private IContentInformationService contentInformationService;*/ @Bean @Scope("prototype") public Logger logger(InjectionPoint injectionPoint) { @@ -208,13 +198,6 @@ public IContentInformationService schemaInformationService() { return new ContentInformationService(); } - @Bean - public IdBasedStorageService idBasedStorageService() { - IdBasedStorageService ibss = new IdBasedStorageService(); - ibss.configure(storageServiceProperties()); - return ibss; - } - @Bean @ConfigurationProperties("repo") public ApplicationProperties applicationProperties() { @@ -262,16 +245,16 @@ public MetastoreConfiguration schemaConfig() { rbc.setEventPublisher(eventPublisher); LOG.trace("Looking for versioningServices...."); for (IRepoVersioningService versioningService : this.versioningServices) { - LOG.trace(".... '{}'", versioningService.getServiceName()); - if ("simple".equals(versioningService.getServiceName())) { + LOG.trace(LIST_ITEM, versioningService.getServiceName()); + if (Objects.equals(versioningService.getServiceName(), DEFAULT_VERSIONING)) { rbc.setVersioningService(versioningService); break; } } LOG.trace("Looking for storageServices...."); for (IRepoStorageService storageService : this.storageServices) { - LOG.trace(".... '{}'", storageService.getServiceName()); - if ("simple".equals(storageService.getServiceName())) { + LOG.trace(LIST_ITEM, storageService.getServiceName()); + if (Objects.equals(storageService.getServiceName(), DEFAULT_STORAGE)) { rbc.setStorageService(storageService); break; } @@ -289,10 +272,12 @@ public MetastoreConfiguration schemaConfig() { MetadataSchemaRecordUtil.setSchemaRecordDao(schemaRecordDao); MetadataSchemaRecordUtil.setMetadataFormatDao(metadataFormatDao); MetadataSchemaRecordUtil.setUrl2PathDao(url2PathDao); + MetadataSchemaRecordUtil.setDataRecordDao(dataRecordDao); fixBasePath(rbc); printSettings(rbc); + LOG.trace("Content audit service: '{}'", contentAuditService); return rbc; } @@ -312,16 +297,21 @@ public MetastoreConfiguration metadataConfig() { rbc.setEventPublisher(eventPublisher); LOG.trace("Looking for versioningServices...."); for (IRepoVersioningService versioningService : this.versioningServices) { - LOG.trace(".... '{}'", versioningService.getServiceName()); - if ("simple".equals(versioningService.getServiceName())) { + LOG.trace(LIST_ITEM, versioningService.getServiceName()); + if (Objects.equals(versioningService.getServiceName(), DEFAULT_VERSIONING)) { rbc.setVersioningService(versioningService); break; } } - LOG.trace("Looking for storageServices...."); + LOG.trace("Looking for storageService '{}'....", this.applicationProperties.getStoragePattern()); for (IRepoStorageService storageService : this.storageServices) { - LOG.trace(".... '{}'", storageService.getServiceName()); - if ("simple".equals(storageService.getServiceName())) { + LOG.trace(LIST_ITEM, storageService.getServiceName()); + if (Objects.equals(storageService.getServiceName(), DEFAULT_STORAGE)) { + rbc.setStorageService(storageService); // Should be used as default + } + if (this.applicationProperties.getStoragePattern().equals(storageService.getServiceName())) { + LOG.trace("Configure '{}' with '{}'", storageService.getServiceName(), storageServiceProperties()); + storageService.configure(storageServiceProperties()); rbc.setStorageService(storageService); break; } @@ -334,10 +324,11 @@ public MetastoreConfiguration metadataConfig() { rbc.setMaxJaversScope(this.applicationProperties.getMaxJaversScope()); rbc.setSchemaRegistries(checkRegistries(applicationProperties.getSchemaRegistries())); rbc.setValidators(validators); - + fixBasePath(rbc); printSettings(rbc); + LOG.trace("Content audit service: '{}'", contentAuditService); return rbc; } @@ -354,10 +345,10 @@ public void printSettings(MetastoreConfiguration config) { LOG.info("Versioning service: {}", config.getVersioningService().getServiceName()); LOG.info("Storage service: {}", config.getStorageService().getServiceName()); LOG.info("Basepath metadata repository: {}", config.getBasepath().toString()); - int noOfSchemaRegistries = config.getSchemaRegistries().length; + int noOfSchemaRegistries = config.getSchemaRegistries().size(); LOG.info("Number of registered external schema registries: {}", noOfSchemaRegistries); for (int index1 = 0; index1 < noOfSchemaRegistries; index1++) { - LOG.info("Schema registry '{}': {}", index1 + 1, config.getSchemaRegistries()[index1]); + LOG.info("Schema registry '{}': {}", index1 + 1, config.getSchemaRegistries().get(index1)); } } @@ -368,15 +359,14 @@ public void printSettings(MetastoreConfiguration config) { * @param currentRegistries Current list of schema registries. * @return Fitered list of schema registries. */ - public String[] checkRegistries(String[] currentRegistries) { + public List checkRegistries(List currentRegistries) { List allRegistries = new ArrayList<>(); for (String schemaRegistry : currentRegistries) { if (!schemaRegistry.trim().isEmpty()) { allRegistries.add(schemaRegistry); } } - String[] array = allRegistries.toArray(new String[0]); - return array; + return allRegistries; } /** @@ -399,8 +389,8 @@ private void fixBasePath(MetastoreConfiguration config) { } public static void main(String[] args) { - ApplicationContext ctx = SpringApplication.run(Application.class, args); - System.out.println("Spring is running!"); + SpringApplication.run(Application.class, args); + LOG.info("Spring is running!"); } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java index 73047615..a37ceae5 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java @@ -15,6 +15,7 @@ */ package edu.kit.datamanager.metastore2.configuration; +import edu.kit.datamanager.annotations.LocalFolderURL; import edu.kit.datamanager.configuration.GenericApplicationProperties; import java.net.URL; import java.util.List; @@ -27,6 +28,7 @@ import org.springframework.validation.annotation.Validated; /** + * Properties for configuration of MetaStore. * * @author jejkal */ @@ -36,23 +38,33 @@ @Validated @RefreshScope @EqualsAndHashCode(callSuper = true) -public class ApplicationProperties extends GenericApplicationProperties{ +public class ApplicationProperties extends GenericApplicationProperties { - @edu.kit.datamanager.annotations.LocalFolderURL + @LocalFolderURL @Value("${metastore.schema.schemaFolder}") private URL schemaFolder; + @Value("${metastore.schema.landingpage:/schema-landing-page?schemaId=$(id)&version=$(version)}") + private String schemaLandingPage; + @Value("${metastore.schema.synchronization.enabled:FALSE}") private boolean synchronizationEnabled; - // @Value("${metastore.schema.synchronization.schemaSources}") + + //@Value("${metastore.schema.synchronization.schemaSources}") private List schemaSources; - @edu.kit.datamanager.annotations.LocalFolderURL + @LocalFolderURL @Value("${metastore.metadata.metadataFolder}") private URL metadataFolder; + @Value("${metastore.metadata.landingpage:/metadata-landing-page?id=$(id)&version=$(version)}") + private String metadataLandingPage; + + @Value("${metastore.metadata.storagepattern:dateBased}") + private String storagePattern; + @Value("${metastore.metadata.schemaRegistries: }") - private String[] schemaRegistries; + private List schemaRegistries; @Value("${metastore.javers.scope:20}") private int maxJaversScope; diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/JPAPersistenceConfig.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/JPAPersistenceConfig.java index 9055930a..a54faf2a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/JPAPersistenceConfig.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/JPAPersistenceConfig.java @@ -21,6 +21,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; /** + * Configuration for JPA persistence. * * @author jejkal */ @@ -28,6 +29,6 @@ @EnableTransactionManagement @EnableJpaRepositories(basePackages = "edu.kit.datamanager.metastore2") @EntityScan(basePackages = {"edu.kit.datamanager.metastore2.domain"}) -public class JPAPersistenceConfig{ +public class JPAPersistenceConfig { } diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/MetastoreConfiguration.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/MetastoreConfiguration.java index cc4dde3f..e182023d 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/MetastoreConfiguration.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/MetastoreConfiguration.java @@ -17,43 +17,51 @@ import edu.kit.datamanager.metastore2.validation.IValidator; import edu.kit.datamanager.repo.configuration.RepoBaseConfiguration; +import java.util.List; /** * Holds all properties needed to manage data resources. */ public class MetastoreConfiguration extends RepoBaseConfiguration { - private String[] schemaRegistries; - - private IValidator[] validators; + private List schemaRegistries; + private Listvalidators; /** + * Get schema registries. + * * @return the schemaRegistries */ - public String[] getSchemaRegistries() { + public List getSchemaRegistries() { return schemaRegistries; } /** + * Set schema registries. + * * @param schemaRegistries the schemaRegistries to set */ - public void setSchemaRegistries(String[] schemaRegistries) { + public void setSchemaRegistries(List schemaRegistries) { this.schemaRegistries = schemaRegistries; } /** + * Get validators for schemas. + * * @return the validators */ - public IValidator[] getValidators() { + public List getValidators() { return validators; } /** + * Set validators for schemas. + * * @param validators the validators to set */ - public void setValidators(IValidator[] validators) { + public void setValidators(List validators) { this.validators = validators; } - + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/OaiPmhConfiguration.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/OaiPmhConfiguration.java index 30023ecd..c476065a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/OaiPmhConfiguration.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/OaiPmhConfiguration.java @@ -25,6 +25,7 @@ import org.springframework.validation.annotation.Validated; /** + * Configuration for OAI-PMH. * * @author jejkal */ @@ -34,7 +35,7 @@ @Validated @RefreshScope @EqualsAndHashCode(callSuper = true) -public class OaiPmhConfiguration extends GenericPluginProperties{ +public class OaiPmhConfiguration extends GenericPluginProperties { @Value("${repo.plugin.oaipmh.adminEmail:none}") private String adminEmail; diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/OpenApiDefinitions.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/OpenApiDefinitions.java index 02c09f84..a83d6779 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/OpenApiDefinitions.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/OpenApiDefinitions.java @@ -1,4 +1,3 @@ - /* * Copyright 2020 Karlsruhe Institute of Technology. * @@ -21,12 +20,11 @@ import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Contact; import io.swagger.v3.oas.models.info.License; -import io.swagger.v3.oas.models.security.SecurityScheme; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** - * + * Configuration for OAI (swagger) * @author jejkal */ @Configuration diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/SpringRestConfiguration.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/SpringRestConfiguration.java index b984bbfd..b90e3748 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/SpringRestConfiguration.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/SpringRestConfiguration.java @@ -22,6 +22,7 @@ import org.springframework.web.servlet.config.annotation.CorsRegistry; /** + * Configuration for spring REST. * * @author jejkal */ diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/SynchronizationSource.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/SynchronizationSource.java index 724ef123..93166fb5 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/SynchronizationSource.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/SynchronizationSource.java @@ -18,11 +18,13 @@ import lombok.Data; /** + * Helper class for synchronizing repositories. * * @author jejkal */ @Data -public class SynchronizationSource{ +@SuppressWarnings("java:S1068") +public class SynchronizationSource { private String id; private String baseUrl; diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/WebSecurityConfig.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/WebSecurityConfig.java index 9ba33e00..9ca308e9 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/WebSecurityConfig.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/WebSecurityConfig.java @@ -17,8 +17,9 @@ import edu.kit.datamanager.security.filter.KeycloakTokenFilter; import edu.kit.datamanager.security.filter.NoAuthenticationFilter; +import edu.kit.datamanager.security.filter.PublicAuthenticationFilter; import java.util.Optional; -import javax.servlet.Filter; +import jakarta.servlet.Filter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -30,12 +31,13 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpMethod; -import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.www.BasicAuthenticationFilter; import org.springframework.security.web.firewall.DefaultHttpFirewall; import org.springframework.security.web.firewall.HttpFirewall; @@ -44,13 +46,14 @@ import org.springframework.web.filter.CorsFilter; /** + * Configuration for web security. * * @author jejkal */ @Configuration @EnableWebSecurity -@EnableGlobalMethodSecurity(prePostEnabled = true) -public class WebSecurityConfig extends WebSecurityConfigurerAdapter { +@EnableMethodSecurity(prePostEnabled = true) +public class WebSecurityConfig { private static final Logger logger = LoggerFactory.getLogger(WebSecurityConfig.class); @@ -65,46 +68,67 @@ public class WebSecurityConfig extends WebSecurityConfigurerAdapter { @Value("${metastore.security.allowedOriginPattern:http*://localhost:*}") private String allowedOriginPattern; + private static final String[] AUTH_WHITELIST_SWAGGER_UI = { + // -- Swagger UI v2 + "/v2/api-docs", + "/swagger-resources", + "/swagger-resources/**", + "/configuration/ui", + "/configuration/security", + "/swagger-ui.html", + "/webjars/**", + // -- Swagger UI v3 (OpenAPI) + "/v3/api-docs/**", + "/swagger-ui/**" + // other public endpoints of your API may be appended to this array + }; + public WebSecurityConfig() { + // Not used } - @Override - protected void configure(HttpSecurity http) throws Exception { - HttpSecurity httpSecurity = http.authorizeRequests(). - antMatchers(HttpMethod.OPTIONS, "/**").permitAll(). - requestMatchers(EndpointRequest.to( - InfoEndpoint.class, - HealthEndpoint.class - )).permitAll(). - requestMatchers(EndpointRequest.toAnyEndpoint()).hasAnyRole("ADMIN", "ACTUATOR"). and(). - sessionManagement(). - sessionCreationPolicy(SessionCreationPolicy.STATELESS).and(); + @Bean + public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + HttpSecurity httpSecurity = http.authorizeHttpRequests( + authorize -> authorize.requestMatchers(HttpMethod.OPTIONS, "/**").permitAll(). + requestMatchers("/oaipmh").permitAll(). + requestMatchers("/static/**").permitAll(). + requestMatchers(AUTH_WHITELIST_SWAGGER_UI).permitAll(). + requestMatchers(EndpointRequest.to( + InfoEndpoint.class, + HealthEndpoint.class + )).permitAll(). + requestMatchers(EndpointRequest.toAnyEndpoint()).hasAnyRole("ANONYMOUS", "ADMIN", "ACTUATOR", "SERVICE_WRITE"). + requestMatchers("/**").authenticated()). + sessionManagement( + session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)); if (!enableCsrf) { logger.info("CSRF disabled!"); - httpSecurity = httpSecurity.csrf().disable(); + httpSecurity = httpSecurity.csrf(csrf -> csrf.disable()); } + logger.info("Adding 'NoAuthenticationFilter' to authentication chain."); if (keycloaktokenFilterBean.isPresent()) { logger.info("Add keycloak filter!"); httpSecurity.addFilterAfter(keycloaktokenFilterBean.get(), BasicAuthenticationFilter.class); + logger.info("Add public authentication filter!"); + httpSecurity = httpSecurity.addFilterAfter(new PublicAuthenticationFilter(applicationProperties.getJwtSecret()), BasicAuthenticationFilter.class); } if (!applicationProperties.isAuthEnabled()) { logger.info("Authentication is DISABLED. Adding 'NoAuthenticationFilter' to authentication chain."); - httpSecurity = httpSecurity.addFilterAfter(new NoAuthenticationFilter(applicationProperties.getJwtSecret(), authenticationManager()), BasicAuthenticationFilter.class); + AuthenticationManager defaultAuthenticationManager = http.getSharedObject(AuthenticationManager.class); + httpSecurity = httpSecurity.addFilterAfter(new NoAuthenticationFilter(applicationProperties.getJwtSecret(), defaultAuthenticationManager), BasicAuthenticationFilter.class); } else { logger.info("Authentication is ENABLED."); } - httpSecurity. - authorizeRequests(). - antMatchers("/api/v1").authenticated(); + httpSecurity.headers(headers -> headers.cacheControl(cache -> cache.disable())); - http.headers().cacheControl().disable(); + return httpSecurity.build(); } - - @Override - public void configure(WebSecurity web) throws Exception { - web.httpFirewall(allowUrlEncodedSlashHttpFirewall()); + @Bean + public WebSecurityCustomizer webSecurityCustomizer() { + return web -> web.httpFirewall(allowUrlEncodedSlashHttpFirewall()); } @Bean @@ -115,7 +139,8 @@ public HttpFirewall allowUrlEncodedSlashHttpFirewall() { } @Bean - public FilterRegistrationBean corsFilter() { + @SuppressWarnings("StringSplitter") + public CorsFilter corsFilter() { final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); CorsConfiguration config = new CorsConfiguration(); config.setAllowCredentials(true); @@ -130,9 +155,8 @@ public FilterRegistrationBean corsFilter() { config.addExposedHeader("ETag"); source.registerCorsConfiguration("/**", config); - FilterRegistrationBean bean; - bean = new FilterRegistrationBean<>(new CorsFilter(source)); - bean.setOrder(0); + CorsFilter bean; + bean = new CorsFilter(source); return bean; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/IDataRecordDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/IDataRecordDao.java index a7b8c7bf..c0f179d2 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/IDataRecordDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/IDataRecordDao.java @@ -1,7 +1,17 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Copyright 2018 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package edu.kit.datamanager.metastore2.dao; @@ -14,7 +24,7 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; /** - * + * DAO for the record of a metadata document. * @author Torridity */ public interface IDataRecordDao extends JpaRepository, JpaSpecificationExecutor{ diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/ILinkedMetadataRecordDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/ILinkedMetadataRecordDao.java index 237ef2fe..3fc52b3c 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/ILinkedMetadataRecordDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/ILinkedMetadataRecordDao.java @@ -11,10 +11,10 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; /** - * - * @author Torridity + * DAO for linked metadata resources. */ -public interface ILinkedMetadataRecordDao extends JpaRepository, JpaSpecificationExecutor{ +public interface ILinkedMetadataRecordDao extends JpaRepository, JpaSpecificationExecutor { + boolean existsMetadataRecordByRelatedResourceAndSchemaId(String relatedResource, String schemaId); - + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataFormatDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataFormatDao.java index 8f7ee6dd..f24f96bf 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataFormatDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataFormatDao.java @@ -1,7 +1,17 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Copyright 2018 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package edu.kit.datamanager.metastore2.dao; @@ -12,8 +22,10 @@ import org.springframework.data.jpa.repository.Query; /** + * DAO for record for OAI-PMH. */ public interface IMetadataFormatDao extends JpaRepository, JpaSpecificationExecutor { + @Query("select p.metadataPrefix from #{#entityName} p") List getAllIds(); diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataSchemaDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataSchemaDao.java index 5ad9f931..42a2d33d 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataSchemaDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/IMetadataSchemaDao.java @@ -10,9 +10,10 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; /** + * DAO for metadata schema record. * * @author Torridity */ -public interface IMetadataSchemaDao extends JpaRepository, JpaSpecificationExecutor{ - +public interface IMetadataSchemaDao extends JpaRepository, JpaSpecificationExecutor { + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaRecordDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaRecordDao.java index 01f10419..7337e000 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaRecordDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaRecordDao.java @@ -1,7 +1,17 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Copyright 2018 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package edu.kit.datamanager.metastore2.dao; @@ -11,13 +21,19 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; /** + * DAO for the record of a schema document. * * @author Torridity */ -public interface ISchemaRecordDao extends JpaRepository, JpaSpecificationExecutor{ +public interface ISchemaRecordDao extends JpaRepository, JpaSpecificationExecutor { + boolean existsSchemaRecordBySchemaIdAndVersion(String schemaId, Long version); + SchemaRecord findBySchemaIdAndVersion(String schemaId, Long version); + List findBySchemaIdOrderByVersionDesc(String schemaId); + SchemaRecord findTopBySchemaIdOrderByVersionDesc(String schemaId); + SchemaRecord findFirstBySchemaIdOrderByVersionDesc(String schemaId); } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaSynchronizationEventDao.java b/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaSynchronizationEventDao.java index fb34f026..c8630596 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaSynchronizationEventDao.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dao/ISchemaSynchronizationEventDao.java @@ -20,10 +20,11 @@ import org.springframework.data.jpa.repository.JpaRepository; /** + * DAO for synchronization events. * * @author jejkal */ -public interface ISchemaSynchronizationEventDao extends JpaRepository{ +public interface ISchemaSynchronizationEventDao extends JpaRepository { public Optional findBySourceName(String sourceName); } diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/AclRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/AclRecord.java index 515b797c..940cc881 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/AclRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/AclRecord.java @@ -18,12 +18,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import edu.kit.datamanager.entities.PERMISSION; import edu.kit.datamanager.repo.domain.acl.AclEntry; +import jakarta.persistence.OneToMany; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.util.HashSet; import java.util.Set; -import javax.persistence.OneToMany; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; import org.springframework.http.MediaType; @@ -34,12 +34,12 @@ @Data public class AclRecord implements Serializable { - public final static String RESOURCE_TYPE = "application/vnd.datamanager.acl+json"; + public static final String RESOURCE_TYPE = "application/vnd.datamanager.acl+json"; - public final static MediaType ACL_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); + public static final MediaType ACL_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); @NotNull(message = "A list of access control entries with at least access for READ.") - @OneToMany(cascade = javax.persistence.CascadeType.ALL, orphanRemoval = true) + @OneToMany(cascade = jakarta.persistence.CascadeType.ALL, orphanRemoval = true) private final Set read; @NotBlank(message = "The metadata record.") private Object metadataRecord; diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/DataRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/DataRecord.java index 80d1983e..b81dca4d 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/DataRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/DataRecord.java @@ -15,19 +15,20 @@ */ package edu.kit.datamanager.metastore2.domain; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.time.Instant; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; /** + * Simplified record for a metadata document. * * @author jejkal */ diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/LinkedMetadataRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/LinkedMetadataRecord.java index db1c5c98..1b3275da 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/LinkedMetadataRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/LinkedMetadataRecord.java @@ -16,40 +16,39 @@ package edu.kit.datamanager.metastore2.domain; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; +import jakarta.validation.constraints.NotBlank; import java.io.Serializable; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; -import javax.validation.constraints.NotBlank; import lombok.Data; /** - * - * @author jejkal + * Simplified record for linked resources for metadata document. */ @Entity @JsonIgnoreProperties(ignoreUnknown = true) @Data -@Table(uniqueConstraints = {@UniqueConstraint(columnNames = {"relatedResource", "schemaId"})}) +@Table(uniqueConstraints = { + @UniqueConstraint(columnNames = {"relatedResource", "schemaId"})}) public class LinkedMetadataRecord implements Serializable { - + public LinkedMetadataRecord() { } - - public LinkedMetadataRecord(MetadataRecord record) { - schemaId = record.getSchema().getIdentifier(); - relatedResource = record.getRelatedResource().getIdentifier(); + + public LinkedMetadataRecord(MetadataRecord metadataRecord) { + schemaId = metadataRecord.getSchema().getIdentifier(); + relatedResource = metadataRecord.getRelatedResource().getIdentifier(); } @Id - @GeneratedValue + @GeneratedValue private Long id; - + @NotBlank(message = "The unqiue identifier of the schema used in the metadata repository for identifying the schema.") private String schemaId; @NotBlank(message = "The unqiue identifier of the related source.") private String relatedResource; - - + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataRecord.java index afeb6896..0f1387b5 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataRecord.java @@ -25,19 +25,20 @@ import edu.kit.datamanager.repo.domain.acl.AclEntry; import edu.kit.datamanager.util.json.CustomInstantDeserializer; import edu.kit.datamanager.util.json.CustomInstantSerializer; +import jakarta.persistence.Id; +import jakarta.persistence.OneToMany; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Set; -import javax.persistence.Id; -import javax.persistence.OneToMany; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; import org.springframework.http.MediaType; /** + * Record for a metadata document. * * @author jejkal */ @@ -45,9 +46,9 @@ @Data public class MetadataRecord implements EtagSupport, Serializable { - public final static String RESOURCE_TYPE = "application/vnd.datamanager.metadata-record+json"; + public static final String RESOURCE_TYPE = "application/vnd.datamanager.metadata-record+json"; - public final static MediaType METADATA_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); + public static final MediaType METADATA_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); @Id @NotBlank(message = "The unique identify of the record.") @@ -74,7 +75,7 @@ public class MetadataRecord implements EtagSupport, Serializable { private Long recordVersion; @NotNull(message = "A list of access control entries for resticting access.") - @OneToMany(cascade = javax.persistence.CascadeType.ALL, orphanRemoval = true) + @OneToMany(cascade = jakarta.persistence.CascadeType.ALL, orphanRemoval = true) private final Set acl = new HashSet<>(); @NotBlank(message = "The metadata document uri, e.g. pointing to a local file.") private String metadataDocumentUri; @@ -89,9 +90,11 @@ public class MetadataRecord implements EtagSupport, Serializable { * @param newAclList new list with acls. */ public void setAcl(Set newAclList) { - acl.clear(); - if (newAclList != null) { - acl.addAll(newAclList); + if (acl != newAclList) { + acl.clear(); + if (newAclList != null) { + acl.addAll(newAclList); + } } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataSchemaRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataSchemaRecord.java index 805c5825..f8793d1d 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataSchemaRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/MetadataSchemaRecord.java @@ -24,25 +24,27 @@ import edu.kit.datamanager.repo.domain.acl.AclEntry; import edu.kit.datamanager.util.json.CustomInstantDeserializer; import edu.kit.datamanager.util.json.CustomInstantSerializer; +import jakarta.persistence.CascadeType; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.OneToOne; +import jakarta.persistence.Transient; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.HashSet; +import java.util.Locale; import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.OneToOne; -import javax.persistence.Transient; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; import org.springframework.http.MediaType; /** + * Record for a metadata document. * * @author jejkal */ @@ -51,9 +53,9 @@ @Data public class MetadataSchemaRecord implements EtagSupport, Serializable { - public final static String RESOURCE_TYPE = "application/vnd.datamanager.schema-record+json"; + public static final String RESOURCE_TYPE = "application/vnd.datamanager.schema-record+json"; - public final static MediaType METADATA_SCHEMA_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); + public static final MediaType METADATA_SCHEMA_RECORD_MEDIA_TYPE = MediaType.valueOf(RESOURCE_TYPE); public enum SCHEMA_TYPE { JSON, @@ -63,8 +65,8 @@ public enum SCHEMA_TYPE { @Id @NotBlank(message = "The unqiue identifier of the schema used in the metadata repository for identifying the schema.") private String schemaId; - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "identifier_id", referencedColumnName = "id") + @OneToOne(cascade = CascadeType.ALL) + @JoinColumn(name = "identifier_id", referencedColumnName = "id") @NotBlank(message = "A globally unique identifier pointing to this record, e.g. DOI, Handle, PURL.") private ResourceIdentifier pid; @NotBlank(message = "The schema version. The version is set by the schema registry and cannot be provided manually. Typically, a new schema version is only for metadata changes via PUT. In a few cases, \"\n" @@ -107,38 +109,57 @@ public enum SCHEMA_TYPE { /** * Set new access control list. + * * @param newAclList new list with acls. */ public void setAcl(Set newAclList) { - acl.clear(); - if (newAclList != null) { - acl.addAll(newAclList); + if (acl != newAclList) { + acl.clear(); + if (newAclList != null) { + acl.addAll(newAclList); + } } } + /** * Set creation date (truncated to milliseconds). + * * @param instant creation date */ public void setCreatedAt(Instant instant) { if (instant != null) { - createdAt = instant.truncatedTo(ChronoUnit.MILLIS); + createdAt = instant.truncatedTo(ChronoUnit.MILLIS); } else { createdAt = null; } } - + /** * Set update date (truncated to milliseconds). + * * @param instant update date */ public void setLastUpdate(Instant instant) { if (instant != null) { - lastUpdate = instant.truncatedTo(ChronoUnit.MILLIS); + lastUpdate = instant.truncatedTo(ChronoUnit.MILLIS); } else { lastUpdate = null; } } + /** + * Set schema ID (transform to lower case). + * + * @param schemaId update date + */ + public void setSchemaId(String schemaId) { + if (schemaId != null) { + this.schemaId = schemaId.toLowerCase(Locale.getDefault()); + } else { + this.schemaId = null; + } + } + @Override @JsonIgnore public String getEtag() { diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/ResourceIdentifier.java b/src/main/java/edu/kit/datamanager/metastore2/domain/ResourceIdentifier.java index a3a0be65..7cc02ea0 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/ResourceIdentifier.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/ResourceIdentifier.java @@ -4,11 +4,11 @@ import java.util.HashMap; import java.util.Map; import com.google.gson.annotations.SerializedName; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.validation.constraints.NotBlank; import java.io.Serializable; import java.util.Arrays; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.validation.constraints.NotBlank; import lombok.Data; @Entity @@ -52,14 +52,25 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other == this) { - return true; - } - if ((other instanceof ResourceIdentifier) == false) { - return false; + boolean returnValue = true; + if (other != this) { + returnValue = false; + if (other instanceof ResourceIdentifier) { + ResourceIdentifier rhs = ((ResourceIdentifier) other); + // check for id + if ((((this.getId() == null) && (rhs.getId() == null)) + || ((this.getId() != null) && this.getId().equals(rhs.getId()))) + //check for identifier + && (((this.getIdentifier() == null) && (rhs.getIdentifier() == null)) + || ((this.getIdentifier() != null) && this.getIdentifier().equals(rhs.getIdentifier()))) + // check for identifierType + && (((this.getIdentifierType() == null) && (rhs.getIdentifierType() == null)) + || ((this.getIdentifierType() != null) && this.getIdentifierType().equals(rhs.getIdentifierType())))) { + returnValue = true; + } + } } - ResourceIdentifier rhs = ((ResourceIdentifier) other); - return ((((this.getIdentifier() == rhs.getIdentifier()) || ((this.getIdentifier() != null) && this.getIdentifier().equals(rhs.getIdentifier())))) && ((this.getIdentifierType() == rhs.getIdentifierType()) || ((this.getIdentifierType() != null) && this.getIdentifierType().equals(rhs.getIdentifierType())))); + return returnValue; } public enum IdentifierType { @@ -106,7 +117,7 @@ public enum IdentifierType { @SerializedName("INTERNAL") INTERNAL("INTERNAL"); private final String value; - private final static Map CONSTANTS = new HashMap(); + private static final Map CONSTANTS = new HashMap<>(); static { for (ResourceIdentifier.IdentifierType c : values()) { @@ -123,10 +134,21 @@ public String toString() { return this.value; } + /** + * Return identifier value as string. + * + * @return identifier value as string. + */ public String value() { return this.value; } + /** + * Get Identifier from value + * + * @param value Value as string. + * @return Identifier + */ public static ResourceIdentifier.IdentifierType fromValue(String value) { ResourceIdentifier.IdentifierType constant = CONSTANTS.get(value); if (constant == null) { @@ -139,6 +161,8 @@ public static ResourceIdentifier.IdentifierType fromValue(String value) { } /** + * Get identifier. + * * @return the identifier */ public String getIdentifier() { @@ -146,6 +170,8 @@ public String getIdentifier() { } /** + * Set identifier. + * * @param identifier the identifier to set */ public void setIdentifier(String identifier) { @@ -153,6 +179,8 @@ public void setIdentifier(String identifier) { } /** + * Get identifier type. + * * @return the identifierType */ public IdentifierType getIdentifierType() { @@ -160,6 +188,8 @@ public IdentifierType getIdentifierType() { } /** + * Set identifier type. + * * @param identifierType the identifierType to set */ public void setIdentifierType(IdentifierType identifierType) { @@ -169,14 +199,33 @@ public void setIdentifierType(IdentifierType identifierType) { this.identifierType = identifierType; } + /** + * Create ResourceIdentifier from URL. + * + * @param identifier Url as string. + * @return Resource identifier. + */ public static ResourceIdentifier factoryUrlResourceIdentifier(String identifier) { return factoryResourceIdentifier(identifier, ResourceIdentifier.IdentifierType.URL); } + /** + * Create ResourceIdentifier from string. + * + * @param identifier Internal identifier as string. + * @return Resource identifier. + */ public static ResourceIdentifier factoryInternalResourceIdentifier(String identifier) { return factoryResourceIdentifier(identifier, ResourceIdentifier.IdentifierType.INTERNAL); } + /** + * Create ResourceIdentifier from string and type. + * + * @param identifier Identifier as string. + * @param type Identifier type of the resource identifier. + * @return Resource identifier. + */ public static ResourceIdentifier factoryResourceIdentifier(String identifier, ResourceIdentifier.IdentifierType type) { ResourceIdentifier resourceIdentifier = new ResourceIdentifier(); resourceIdentifier.setIdentifier(identifier); diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaRecord.java b/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaRecord.java index d0ec3ee8..bc04b610 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaRecord.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaRecord.java @@ -15,20 +15,21 @@ */ package edu.kit.datamanager.metastore2.domain; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; /** + * Simplified record for a schema document. * * @author jejkal */ diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaSynchronizationEvent.java b/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaSynchronizationEvent.java index c2ec8616..b22cdcb4 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaSynchronizationEvent.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/SchemaSynchronizationEvent.java @@ -15,20 +15,21 @@ */ package edu.kit.datamanager.metastore2.domain; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; import java.time.Instant; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; import lombok.Data; /** + * Event for synchronization. * * @author jejkal */ @Entity @Data -public class SchemaSynchronizationEvent{ +public class SchemaSynchronizationEvent { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/Url2Path.java b/src/main/java/edu/kit/datamanager/metastore2/domain/Url2Path.java index 489e2cba..43f254e9 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/Url2Path.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/Url2Path.java @@ -16,16 +16,17 @@ package edu.kit.datamanager.metastore2.domain; import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord.SCHEMA_TYPE; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import java.io.Serializable; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Id; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import lombok.Data; /** + * Entity for holding internal path information for given URL. * * @author jejkal */ diff --git a/src/main/java/edu/kit/datamanager/metastore2/domain/oaipmh/MetadataFormat.java b/src/main/java/edu/kit/datamanager/metastore2/domain/oaipmh/MetadataFormat.java index 3f2502e0..c182a1ed 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/domain/oaipmh/MetadataFormat.java +++ b/src/main/java/edu/kit/datamanager/metastore2/domain/oaipmh/MetadataFormat.java @@ -16,13 +16,18 @@ package edu.kit.datamanager.metastore2.domain.oaipmh; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.validation.constraints.NotBlank; import java.io.Serializable; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.validation.constraints.NotBlank; import lombok.Data; /** + * Record for OAI-PMH holding + *
    + *
  • prefix
    • + *
  • schema
    • + *
  • namespace
* * @author jejkal */ diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestMetadata.java b/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestMetadata.java index 4806d8a2..5e82fd3c 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestMetadata.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestMetadata.java @@ -12,30 +12,32 @@ import org.json.simple.JSONObject; /** + * Data transfer object for Web UI. * * @author sabrinechelbi */ @Builder @Getter +@SuppressWarnings("java:S1068") public class EditorRequestMetadata { - - /** - * JSON schema, which describes the structure of the data model. - */ - private JSONObject dataModel; - - /** - * JSON user interface form, which describes the structure of the form layout. - */ - private JSONObject uiForm; - - /** - * array of schema records. - */ - private List metadataRecords; - - /** - * array, which includes the table’s column definitions. - */ - private TabulatorItems[] items; + + /** + * JSON schema, which describes the structure of the data model. + */ + private JSONObject dataModel; + + /** + * JSON user interface form, which describes the structure of the form layout. + */ + private JSONObject uiForm; + + /** + * array of schema records. + */ + private List metadataRecords; + + /** + * array, which includes the table’s column definitions. + */ + private TabulatorItems[] items; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestSchema.java b/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestSchema.java index 35c5bd4c..fc30a060 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestSchema.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/EditorRequestSchema.java @@ -12,32 +12,33 @@ import org.json.simple.JSONObject; /** + * Data transfer object for Web UI. * * @author sabrinechelbi */ @Builder @Getter +@SuppressWarnings("java:S1068") public class EditorRequestSchema { - /** - * JSON schema, which describes the structure of the data model. - */ - private JSONObject dataModel; + /** + * JSON schema, which describes the structure of the data model. + */ + private JSONObject dataModel; - /** - * JSON user interface form, which describes the structure of the form - * layout. - */ - private JSONObject uiForm; + /** + * JSON user interface form, which describes the structure of the form layout. + */ + private JSONObject uiForm; - /** - * array of schema records. - */ - private List schemaRecords; + /** + * array of schema records. + */ + private List schemaRecords; - /** - * array, which includes the table’s column definitions. - */ - private TabulatorItems[] items; + /** + * array, which includes the table’s column definitions. + */ + private TabulatorItems[] items; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorFormatterParam.java b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorFormatterParam.java index a9161fce..fe00f815 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorFormatterParam.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorFormatterParam.java @@ -8,10 +8,13 @@ import lombok.Getter; /** + * Helper class for tabulators. * * @author sabrinechelbi */ @Getter +@SuppressWarnings("java:S1068") public class TabulatorFormatterParam { - private String outputFormat; + + private String outputFormat; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorItems.java b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorItems.java index ca6a594c..090b6db8 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorItems.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorItems.java @@ -8,27 +8,28 @@ import lombok.Getter; /** + * Helper class for tabulator items. * * @author sabrinechelbi */ @Getter - +@SuppressWarnings("java:S1068") public class TabulatorItems { - - /** - * The title that will be displayed in the header for the column. - */ - private String title; - - /** - * The key for the column in the JSON resource. - */ - private String field; - - private String editor; - - private String formatter; - - private TabulatorFormatterParam formatterParams; - + + /** + * The title that will be displayed in the header for the column. + */ + private String title; + + /** + * The key for the column in the JSON resource. + */ + private String field; + + private String editor; + + private String formatter; + + private TabulatorFormatterParam formatterParams; + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorLocalPagination.java b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorLocalPagination.java index de0b9115..4a3fdb15 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorLocalPagination.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorLocalPagination.java @@ -10,11 +10,13 @@ import lombok.Getter; /** + * Helper class for tabulator local pagination used by web frontend. * * @author jejkal */ @Getter @Builder +@SuppressWarnings("java:S1068") public class TabulatorLocalPagination { @JsonProperty("last_page") private int lastPage; diff --git a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorRemotePagination.java b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorRemotePagination.java index b0e6387a..109d21c7 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorRemotePagination.java +++ b/src/main/java/edu/kit/datamanager/metastore2/dto/TabulatorRemotePagination.java @@ -11,14 +11,17 @@ import lombok.Getter; /** + * Helper class for tabulator remote pagination used by web frontend. * * @author sabrinechelbi */ @Getter @Builder +@SuppressWarnings("java:S1068") public class TabulatorRemotePagination { - @JsonProperty("last_page") - private int lastPage; - - private List data; + + @JsonProperty("last_page") + private int lastPage; + + private List data; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/filter/AccessLoggingFilter.java b/src/main/java/edu/kit/datamanager/metastore2/filter/AccessLoggingFilter.java index e42ac82c..d26a8a07 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/filter/AccessLoggingFilter.java +++ b/src/main/java/edu/kit/datamanager/metastore2/filter/AccessLoggingFilter.java @@ -16,14 +16,14 @@ package edu.kit.datamanager.metastore2.filter; import edu.kit.datamanager.security.filter.KeycloakTokenFilter; +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/AbstractOAIPMHRepository.java b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/AbstractOAIPMHRepository.java index 2037d493..aba3f202 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/AbstractOAIPMHRepository.java +++ b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/AbstractOAIPMHRepository.java @@ -40,6 +40,8 @@ public abstract class AbstractOAIPMHRepository { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AbstractOAIPMHRepository.class); + + private static final String VERSION = "2.0"; private String name = null; private DateFormat df = null; @@ -49,7 +51,7 @@ public abstract class AbstractOAIPMHRepository { * * @param repositoryName The repository name. */ - public AbstractOAIPMHRepository(String repositoryName) { + protected AbstractOAIPMHRepository(String repositoryName) { name = repositoryName; } @@ -212,7 +214,7 @@ public String getRepositoryName() { * @return The supported OAI-PMH protocol version. */ public String getProtocolVersion() { - return "2.0"; + return VERSION; } /** @@ -225,9 +227,9 @@ public DateFormat getDateFormat() { switch (getGranularity()) { case YYYY_MM_DD: df = new SimpleDateFormat("yyyy-MM-dd"); + break; default: df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - break; } df.setTimeZone(TimeZone.getTimeZone("UTC")); } diff --git a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java index 79fdecd9..23f0ed38 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java +++ b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java @@ -36,6 +36,7 @@ import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -45,9 +46,10 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; +import java.util.function.UnaryOperator; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -64,6 +66,9 @@ import org.purl.dc.elements._1.ElementContainer; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Example; +import org.springframework.data.domain.ExampleMatcher; +import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; import org.springframework.web.servlet.support.ServletUriComponentsBuilder; @@ -99,6 +104,7 @@ * @author jejkal */ @Component +@SuppressWarnings("JavaUtilDate") public class MetastoreOAIPMHRepository extends AbstractOAIPMHRepository { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(MetastoreOAIPMHRepository.class); @@ -108,15 +114,11 @@ public class MetastoreOAIPMHRepository extends AbstractOAIPMHRepository { private OaiPmhConfiguration pluginConfiguration; - @Autowired - private ApplicationProperties metastoreProperties; @Autowired private IDataRecordDao dataRecordDao; @Autowired private IMetadataFormatDao metadataFormatDao; @Autowired - private OaiPmhConfiguration oaiConfigProperties; - @Autowired private MetastoreConfiguration metadataConfig; /** @@ -181,13 +183,15 @@ public boolean isPrefixSupported(String prefix) { boolean exists = DC_SCHEMA.getMetadataPrefix().equals(prefix) || DATACITE_SCHEMA.getMetadataPrefix().equals(prefix); LOGGER.trace(prefix + ": " + exists); if (!exists) { - List findAll = metadataFormatDao.findAll(); - for (MetadataFormat item : findAll) { - LOGGER.trace("." + item.getMetadataPrefix()); - if (prefix.equalsIgnoreCase(item.getMetadataPrefix())) { - exists = true; - break; - } + MetadataFormat metadataFormat = new MetadataFormat(); + metadataFormat.setMetadataPrefix(prefix); + ExampleMatcher caseInsensitive = ExampleMatcher.matchingAll().withIgnoreCase(); + Example example = Example.of(metadataFormat, caseInsensitive); + Optional findOne = metadataFormatDao.findOne(example); + if (findOne.isPresent()) { + MetadataFormat item = findOne.get(); + LOGGER.trace("Found at least one item with prefix: " + item.getMetadataPrefix()); + exists = true; } } return exists; @@ -213,16 +217,21 @@ public void listMetadataFormats(OAIPMHBuilder builder) { //@TODO extend by other formats builder.addMetadataFormat(DC_SCHEMA); builder.addMetadataFormat(DATACITE_SCHEMA); - List allXmlSchemas = metadataFormatDao.findAll(); + long noOfEntries = metadataFormatDao.count(); + long entriesPerPage = 50; + long page = 0; // add also the schema registered in the schema registry - for (MetadataFormat metadataFormat : allXmlSchemas) { - MetadataFormatType item = new MetadataFormatType(); - item.setMetadataNamespace(metadataFormat.getMetadataNamespace()); - item.setMetadataPrefix(metadataFormat.getMetadataPrefix()); - item.setSchema(metadataFormat.getSchema()); - builder.addMetadataFormat(item); - } - + do { + List allXmlSchemas = metadataFormatDao.findAll(PageRequest.of((int) page, (int) entriesPerPage)).getContent(); + for (MetadataFormat metadataFormat : allXmlSchemas) { + MetadataFormatType item = new MetadataFormatType(); + item.setMetadataNamespace(metadataFormat.getMetadataNamespace()); + item.setMetadataPrefix(metadataFormat.getMetadataPrefix()); + item.setSchema(metadataFormat.getSchema()); + builder.addMetadataFormat(item); + } + page++; + } while (page * entriesPerPage < noOfEntries); } @Override @@ -238,7 +247,7 @@ public void listIdentifiers(OAIPMHBuilder builder) { } LOGGER.trace("Adding {} records to result.", results.size()); - results.stream().forEach((result) -> { + results.stream().forEach(result -> { //TODO get proper date Date changeDate = new Date(0l); if (result.getLastUpdate() != null) { @@ -277,9 +286,7 @@ public void listRecords(OAIPMHBuilder builder) { return; } LOGGER.trace("Adding {} records to result.", results.size()); - results.stream().forEach((result) -> { - addRecordEntry(result, builder); - }); + results.stream().forEach(result -> addRecordEntry(result, builder)); } /** @@ -307,10 +314,8 @@ private Document getMetadataDocument(DataRecord object, String schemaId) { LOGGER.info("Creating Dublin Core document on the fly.", object.getId()); //create DC metadata try { - Function dummy; - dummy = (t) -> { - return "dummy" + t; - }; + UnaryOperator dummy; + dummy = t -> "dummy" + t; edu.kit.datamanager.repo.domain.DataResource dr = DataResourceUtils.getResourceByIdentifierOrRedirect(metadataConfig, object.getMetadataId(), null, dummy); ElementContainer container = DublinCoreMapper.dataResourceToDublinCoreContainer(DataResourceUtils.migrateToDataResource(dr)); JAXBContext jaxbContext = JAXBContext.newInstance(ElementContainer.class); @@ -324,10 +329,8 @@ private Document getMetadataDocument(DataRecord object, String schemaId) { } else if (DATACITE_SCHEMA.getMetadataPrefix().equals(schemaId)) { LOGGER.info("Creating Datacite document on the fly.", object.getId()); try { - Function dummy; - dummy = (t) -> { - return "dummy" + t; - }; + UnaryOperator dummy; + dummy = t -> "dummy" + t; edu.kit.datamanager.repo.domain.DataResource dr = DataResourceUtils.getResourceByIdentifierOrRedirect(metadataConfig, object.getMetadataId(), null, dummy); // Todo check for internal related schema identifier switch to URL Resource resource = DataCiteMapper.dataResourceToDataciteResource(DataResourceUtils.migrateToDataResource(dr)); @@ -429,6 +432,7 @@ private DataRecord getEntity(OAIPMHBuilder builder) { * * @return A list of entities which might be empty. */ + @SuppressWarnings("StringSplitter") private List getEntities(OAIPMHBuilder builder) { List results; @@ -442,27 +446,23 @@ private List getEntities(OAIPMHBuilder builder) { Instant until = builder.getUntilDate() != null ? builder.getUntilDate().toInstant() : Instant.now().plus(1, ChronoUnit.SECONDS); //check resumption token if (resumptionToken != null) { + String tokenValue = new String(Base64.decodeBase64(URLDecoder.decode(resumptionToken, StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + LOGGER.trace("Found token with value {}", tokenValue); + String[] elements = tokenValue.split("/"); + if (elements.length != 2) { + LOGGER.error("Invalid resumption token. Returning OAI-PMH error BAD_RESUMPTION_TOKEN."); + builder.addError(OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN, null); + return new ArrayList<>(); + } try { - String tokenValue = new String(Base64.decodeBase64(URLDecoder.decode(resumptionToken, "UTF-8"))); - LOGGER.trace("Found token with value {}", tokenValue); - String[] elements = tokenValue.split("/"); - if (elements.length != 2) { - LOGGER.error("Invalid resumption token. Returning OAI-PMH error BAD_RESUMPTION_TOKEN."); - builder.addError(OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN, null); - return new ArrayList<>(); - } - try { - LOGGER.trace("Parsing token values."); - currentCursor = Integer.parseInt(elements[0]); - overallCount = Integer.parseInt(elements[1]); - LOGGER.trace("Obtained {} as current cursor from token. Overall element count is {}.", currentCursor, overallCount); - } catch (NumberFormatException ex) { - //log error - builder.addError(OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN, null); - return new ArrayList<>(); - } - } catch (UnsupportedEncodingException ex) { - LOGGER.error("Failed to get results from repository. Returning empty list.", ex); + LOGGER.trace("Parsing token values."); + currentCursor = Integer.parseInt(elements[0]); + overallCount = Integer.parseInt(elements[1]); + LOGGER.trace("Obtained {} as current cursor from token. Overall element count is {}.", currentCursor, overallCount); + } catch (NumberFormatException ex) { + //log error + builder.addError(OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN, null); + return new ArrayList<>(); } } else { LOGGER.trace("No resumption token found."); @@ -482,23 +482,23 @@ private List getEntities(OAIPMHBuilder builder) { LOGGER.trace("findBySchemaIdAndLastUpdateBetween({},{},{}, Page({},{}))", findMetadataPrefix, from, until, page, maxElementsPerList); overallCount = dataRecordDao.countBySchemaIdInAndLastUpdateBetween(findMetadataPrefix, from, until); results = dataRecordDao.findBySchemaIdInAndLastUpdateBetween(findMetadataPrefix, from, until, PageRequest.of(page, maxElementsPerList)); - LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.findAll().size() + "' elements in total!"); + LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.count() + "' elements in total!"); } else { LOGGER.trace("findBySchemaIdAndLastUpdateBetween({},{},{}, Page({},{}))", prefix, from, until, page, maxElementsPerList); overallCount = dataRecordDao.countBySchemaIdAndLastUpdateBetween(prefix, from, until); results = dataRecordDao.findBySchemaIdAndLastUpdateBetween(prefix, from, until, PageRequest.of(page, maxElementsPerList)); - LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.findAll().size() + "' elements in total!"); + LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.count() + "' elements in total!"); } if (LOGGER.isTraceEnabled()) { - LOGGER.trace("List all items:"); - List findAll = dataRecordDao.findAll(); + LOGGER.trace("List top 100 of all items:"); + List findAll = dataRecordDao.findAll(PageRequest.of(0, 100)).getContent(); for (DataRecord item : findAll) { LOGGER.trace("-> " + item); } } LOGGER.trace("Setting next resumption token."); int cursor = currentCursor + results.size(); - + if (cursor == overallCount) { LOGGER.debug("New cursor {} exceeds element count {}, no more elements available. Setting resumption token to 'null'.", (currentCursor + maxElementsPerList), overallCount); //lsit complete, add no resumptiontoken @@ -513,12 +513,8 @@ private List getEntities(OAIPMHBuilder builder) { //we set no expiration as the token never expires String value = token.getCursor().intValue() + "/" + token.getCompleteListSize().intValue(); LOGGER.trace("Setting resumption token value to {}.", value); - try { - token.setValue(URLEncoder.encode(Base64.encodeBase64String(value.getBytes()), "UTF-8")); - builder.setResumptionToken(token); - } catch (UnsupportedEncodingException ex) { - LOGGER.error("Failed to get results from repository. Returning empty list.", ex); - } + token.setValue(URLEncoder.encode(Base64.encodeBase64String(value.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); + builder.setResumptionToken(token); } return results; diff --git a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/util/OAIPMHBuilder.java b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/util/OAIPMHBuilder.java index 9cfde5ce..f3ee08f7 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/util/OAIPMHBuilder.java +++ b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/util/OAIPMHBuilder.java @@ -46,7 +46,7 @@ /** * Helper class for collecting request parameters and building OAI-PMH response. */ -public class OAIPMHBuilder{ +public class OAIPMHBuilder { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(OAIPMHBuilder.class); @@ -62,34 +62,34 @@ public class OAIPMHBuilder{ private ListIdentifiersType listIdentifiers; private ListRecordsType listRecordsType; - public OAIPMHBuilder(){ + public OAIPMHBuilder() { response = new OAIPMHtype(); } - public static OAIPMHBuilder init(AbstractOAIPMHRepository repository, VerbType verb, String metadataPrefix, String identifier, Date from, Date until, String resumptionToken){ + public static OAIPMHBuilder init(AbstractOAIPMHRepository repository, VerbType verb, String metadataPrefix, String identifier, Date from, Date until, String resumptionToken) { return new OAIPMHBuilder().initRequest(repository, verb, metadataPrefix, identifier, from, until, resumptionToken); } - OAIPMHBuilder initRequest(AbstractOAIPMHRepository repository, VerbType verb, String metadataPrefix, String identifier, Date from, Date until, String resumptionToken){ + OAIPMHBuilder initRequest(AbstractOAIPMHRepository repository, VerbType verb, String metadataPrefix, String identifier, Date from, Date until, String resumptionToken) { this.repository = repository; requestType = new RequestType(); - if(from != null){ + if (from != null) { requestType.setFrom(repository.getDateFormat().format(from)); } - if(until != null){ + if (until != null) { requestType.setUntil(repository.getDateFormat().format(until)); } requestType.setIdentifier(identifier); requestType.setMetadataPrefix(metadataPrefix); requestType.setValue(repository.getBaseUrl()); requestType.setVerb(verb); - if(verb == null){ + if (verb == null) { addError(OAIPMHerrorcodeType.BAD_VERB, "Invalid verb type provided."); return this; } this.verb = verb; - switch(verb){ + switch (verb) { case IDENTIFY: //no mandatory arguments identifyType = new IdentifyType(); @@ -112,21 +112,21 @@ OAIPMHBuilder initRequest(AbstractOAIPMHRepository repository, VerbType verb, St break; case GET_RECORD: //identifier and metadata prefix are mandatory - if(identifier == null || metadataPrefix == null){ + if (identifier == null || metadataPrefix == null) { addError(OAIPMHerrorcodeType.BAD_ARGUMENT, "Arguments identifier and metadataPrefix must not be null while using verb " + verb + "."); } getRecordType = new GetRecordType(); break; case LIST_IDENTIFIERS: - if(metadataPrefix == null){ + if (metadataPrefix == null) { addError(OAIPMHerrorcodeType.BAD_ARGUMENT, "Argument metadataPrefix must not be null while using verb " + verb + "."); } this.resumptionToken = resumptionToken; listIdentifiers = new ListIdentifiersType(); break; case LIST_RECORDS: - if(metadataPrefix == null){ + if (metadataPrefix == null) { addError(OAIPMHerrorcodeType.BAD_ARGUMENT, "Argument metadataPrefix must not be null while using verb " + verb + "."); } this.resumptionToken = resumptionToken; @@ -136,141 +136,145 @@ OAIPMHBuilder initRequest(AbstractOAIPMHRepository repository, VerbType verb, St return this; } - public Date getFromDate(){ + public Date getFromDate() { String date = requestType.getFrom(); - try{ - if(date != null){ + try { + if (date != null) { return repository.getDateFormat().parse(date); } - } catch(ParseException ex){ + } catch (ParseException ex) { //wrong date } return null; } - public Date getUntilDate(){ + public Date getUntilDate() { String date = requestType.getUntil(); - try{ - if(date != null){ + try { + if (date != null) { return repository.getDateFormat().parse(date); } - } catch(ParseException ex){ + } catch (ParseException ex) { //wrong date } return null; } - public VerbType getVerb(){ + public VerbType getVerb() { return requestType.getVerb(); } - public String getMetadataPrefix(){ + public String getMetadataPrefix() { return requestType.getMetadataPrefix(); } - public String getIdentifier(){ + public String getIdentifier() { return requestType.getIdentifier(); } - public String getResumptionToken(){ + public String getResumptionToken() { return resumptionToken; } - public OAIPMHBuilder addMetadataFormat(MetadataFormatType format){ - if(isError()){ + public OAIPMHBuilder addMetadataFormat(MetadataFormatType format) { + if (isError()) { return this; } - switch(verb){ + switch (verb) { case LIST_METADATA_FORMATS: listMetadataFormatsType.getMetadataFormat().add(format); break; + default: + // no action required } return this; } - public OAIPMHBuilder addSet(String name, String spec){ - if(isError()){ + public OAIPMHBuilder addSet(String name, String spec) { + if (isError()) { return this; } return addSet(name, spec, null); } - public OAIPMHBuilder addSet(String name, String spec, Object description){ - if(isError()){ + public OAIPMHBuilder addSet(String name, String spec, Object description) { + if (isError()) { return this; } - switch(verb){ + switch (verb) { case LIST_SETS: SetType set = new SetType(); set.setSetName(name); set.setSetSpec(spec); - if(description != null){ + if (description != null) { DescriptionType descriptionType = new DescriptionType(); descriptionType.setAny(description); set.getSetDescription().add(descriptionType); } listSetsType.getSet().add(set); break; + default: + // no action required } return this; } - public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs){ - if(isError()){ + public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs) { + if (isError()) { return this; } return addRecord(identifier, recordDatestamp, setSpecs, null, null); } - public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs, Object metadata){ - if(isError()){ + public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs, Object metadata) { + if (isError()) { return this; } return addRecord(identifier, recordDatestamp, setSpecs, metadata, null); } - public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs, Object metadata, Object about){ - if(isError()){ + public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List setSpecs, Object metadata, Object about) { + if (isError()) { return this; } - switch(verb){ + switch (verb) { case GET_RECORD: case LIST_RECORDS: { - RecordType record = new RecordType(); + RecordType recordType = new RecordType(); HeaderType header = new HeaderType(); - if(identifier == null || recordDatestamp == null){ + if (identifier == null || recordDatestamp == null) { throw new IllegalArgumentException("Arguments identifier and recordDatestamp must not be null."); } header.setIdentifier(identifier); header.setDatestamp(repository.getDateFormat().format(recordDatestamp)); - if(setSpecs != null){ + if (setSpecs != null) { header.getSetSpec().addAll(setSpecs); } - record.setHeader(header); - if(metadata != null){ + recordType.setHeader(header); + if (metadata != null) { //record metadata MetadataType md = new MetadataType(); md.setAny(metadata); - record.setMetadata(md); + recordType.setMetadata(md); } - if(about != null){ + if (about != null) { //about info defined by community AboutType aboutType = new AboutType(); aboutType.setAny(about); - record.getAbout().add(aboutType); + recordType.getAbout().add(aboutType); } - if(VerbType.GET_RECORD.equals(verb)){ - getRecordType.setRecord(record); - } else{ - listRecordsType.getRecord().add(record); + if (VerbType.GET_RECORD.equals(verb)) { + getRecordType.setRecord(recordType); + } else { + listRecordsType.getRecord().add(recordType); } break; } case LIST_IDENTIFIERS: { HeaderType header = new HeaderType(); - if(identifier == null || recordDatestamp == null){ + if (identifier == null || recordDatestamp == null) { throw new IllegalArgumentException("Arguments identifier and recordDatestamp must not be null."); } header.setIdentifier(identifier); @@ -280,25 +284,27 @@ public OAIPMHBuilder addRecord(String identifier, Date recordDatestamp, List indices; + Set indices; @Parameter(names = {"--updateDate", "-u"}, description = "Starting reindexing only for documents updated at earliest on update date.") Date updateDate; @@ -73,6 +76,7 @@ public class ElasticIndexerRunner implements CommandLineRunner { private Optional messagingService; @Override + @SuppressWarnings({"StringSplitter", "JavaUtilDate"}) public void run(String... args) throws Exception { JCommander argueParser = JCommander.newBuilder() .addObject(this) @@ -83,7 +87,7 @@ public void run(String... args) throws Exception { updateDate = new Date(0); } if (indices == null) { - indices = new ArrayList<>(); + indices = new HashSet<>(); } } catch (Exception ex) { argueParser.usage(); @@ -91,8 +95,9 @@ public void run(String... args) throws Exception { } if (updateIndex) { LOG.info("Start ElasticIndexer Runner for indices '{}' and update date '{}'", indices, updateDate); + LOG.info("No of schemas: '{}'", schemaRecordDao.count()); // Try to determine URL of repository - List findAllSchemas = schemaRecordDao.findAll(); + List findAllSchemas = schemaRecordDao.findAll(PageRequest.of(0, 3)).getContent(); if (!findAllSchemas.isEmpty()) { // There is at least one schema. // Try to fetch baseURL from this @@ -103,14 +108,26 @@ public void run(String... args) throws Exception { if (indices.isEmpty()) { LOG.info("Reindex all indices!"); - for (SchemaRecord item : findAllSchemas) { - indices.add(item.getSchemaId()); - } + long noOfEntries = url2PathDao.count(); + long entriesPerPage = 50; + long page = 0; + // add also the schema registered in the schema registry + do { + List allSchemas = schemaRecordDao.findAll(PageRequest.of((int) page, (int) entriesPerPage)).getContent(); + LOG.trace("Add '{}' schemas of '{}'", allSchemas.size(), noOfEntries); + for (SchemaRecord item : allSchemas) { + indices.add(item.getSchemaId()); + } + page++; + } while (page * entriesPerPage < noOfEntries); } for (String index : indices) { LOG.info("Reindex '{}'", index); List findBySchemaId = dataRecordDao.findBySchemaIdAndLastUpdateAfter(index, updateDate.toInstant()); + List findSchemaBySchemaId = schemaRecordDao.findBySchemaIdOrderByVersionDesc(index); + LOG.trace("Search for documents for schema '{}' and update date '{}'", index, updateDate); + LOG.trace("No of documents: '{}'", findBySchemaId.size()); for (DataRecord item : findBySchemaId) { MetadataRecord result = toMetadataRecord(item, baseUrl); LOG.trace("Sending CREATE event."); @@ -119,13 +136,15 @@ public void run(String... args) throws Exception { } LOG.trace("Search for alternative schemaId (given as URL)"); DataRecord templateRecord = new DataRecord(); - for (SchemaRecord debug : schemaRecordDao.findBySchemaIdOrderByVersionDesc(index)) { + for (SchemaRecord debug : findSchemaBySchemaId) { templateRecord.setSchemaId(debug.getSchemaId()); templateRecord.setSchemaVersion(debug.getVersion()); List findByPath1 = url2PathDao.findByPath(debug.getSchemaDocumentUri()); for (Url2Path path : findByPath1) { LOG.trace("SchemaRecord: '{}'", debug); List findBySchemaUrl = dataRecordDao.findBySchemaIdAndLastUpdateAfter(path.getUrl(), updateDate.toInstant()); + LOG.trace("Search for documents for schema '{}' and update date '{}'", path.getUrl(), updateDate); + LOG.trace("No of documents: '{}'", findBySchemaUrl.size()); for (DataRecord item : findBySchemaUrl) { templateRecord.setMetadataId(item.getMetadataId()); templateRecord.setVersion(item.getVersion()); @@ -144,13 +163,35 @@ public void run(String... args) throws Exception { } } - private MetadataRecord toMetadataRecord(DataRecord record, String baseUrl) { - String metadataIdWithVersion = baseUrl + WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(MetadataControllerImpl.class).getMetadataDocumentById(record.getMetadataId(), record.getVersion(), null, null)).toUri().toString(); + /** + * Transform DataRecord to MetadataRecord. + * + * @param dataRecord DataRecord holding all information about metadata document. + * @param baseUrl Base URL for accessing service. + * @return MetadataRecord of metadata document. + */ + private MetadataRecord toMetadataRecord(DataRecord dataRecord, String baseUrl) { + String metadataIdWithVersion = baseUrl + WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(MetadataControllerImpl.class).getMetadataDocumentById(dataRecord.getMetadataId(), dataRecord.getVersion(), null, null)).toUri().toString(); MetadataRecord returnValue = new MetadataRecord(); - returnValue.setId(metadataIdWithVersion); + returnValue.setId(dataRecord.getMetadataId()); + returnValue.setSchemaVersion(dataRecord.getSchemaVersion()); + returnValue.setRecordVersion(dataRecord.getVersion()); returnValue.setMetadataDocumentUri(metadataIdWithVersion); - returnValue.setSchema(ResourceIdentifier.factoryUrlResourceIdentifier(record.getSchemaId())); - LOG.trace("MetadataRecord: '{}'", returnValue); + returnValue.setSchema(ResourceIdentifier.factoryUrlResourceIdentifier(toSchemaUrl(dataRecord, baseUrl))); + return returnValue; } + + /** + * Transform schemaID to URL if it is an internal + * + * @param dataRecord DataRecord holding schemaID and schema version. + * @param baseUrl Base URL for accessing service. + * @return URL to Schema as String. + */ + private String toSchemaUrl(DataRecord dataRecord, String baseUrl) { + String schemaUrl; + schemaUrl = baseUrl + WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(SchemaRegistryControllerImpl.class).getSchemaDocumentById(dataRecord.getSchemaId(), dataRecord.getVersion(), null, null)).toUri(); + return schemaUrl; + } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/service/SchemaSynchronizationService.java b/src/main/java/edu/kit/datamanager/metastore2/service/SchemaSynchronizationService.java index b2f7e255..b214c84a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/service/SchemaSynchronizationService.java +++ b/src/main/java/edu/kit/datamanager/metastore2/service/SchemaSynchronizationService.java @@ -46,13 +46,16 @@ import org.springframework.web.util.UriComponentsBuilder; /** + * Service for snchronizing repositories. * * @author jejkal */ @Component -public class SchemaSynchronizationService{ +public class SchemaSynchronizationService { private static final Logger LOGGER = LoggerFactory.getLogger(SchemaSynchronizationService.class); + + private static final String LOG_SYNCHRONIZATION_FINISHED = "Schema synchronization finished."; @Autowired private ApplicationProperties applicationProperties; @@ -65,14 +68,14 @@ public class SchemaSynchronizationService{ private MetastoreConfiguration schemaConfig; @Scheduled(cron = "${repo.schema.synchronization.cron.value:-}") - public void performSynchronization(){ + public void performSynchronization() { - if(!applicationProperties.isSynchronizationEnabled()){ + if (!applicationProperties.isSynchronizationEnabled()) { LOGGER.trace("Schema synchonization is disabled."); return; } - if(applicationProperties.getSchemaSources() == null || applicationProperties.getSchemaSources().isEmpty()){ + if (applicationProperties.getSchemaSources() == null || applicationProperties.getSchemaSources().isEmpty()) { LOGGER.warn("Schema synchonization is enabled but schema sources are empty. Skipping schema synchronization."); return; } @@ -81,27 +84,27 @@ public void performSynchronization(){ String localPort = environment.getProperty("local.server.port"); boolean isSecure = (environment.getProperty("server.ssl.key-store") != null); - String localBaseUrl = (isSecure) ? "https://localhost:" : "http://localhost:"; + String localBaseUrl = isSecure ? "https://localhost:" : "http://localhost:"; localBaseUrl += localPort; localBaseUrl += "/api/v1/schemas/"; LOGGER.trace("Using {} as local base URL.", localBaseUrl); LOGGER.trace("Synchronizing schema(s) from {} source(s).", applicationProperties.getSchemaSources().size()); RestTemplate restTemplate = new RestTemplate(); - for(SynchronizationSource source : applicationProperties.getSchemaSources()){ + for (SynchronizationSource source : applicationProperties.getSchemaSources()) { LOGGER.trace("Synchronizing schema(s) from source '{}'.", source.getId()); Optional optEvent = schemaSynchronizationEventDao.findBySourceName(source.getId()); Instant lastSynchronization = null; SchemaSynchronizationEvent event; - if(optEvent.isPresent()){ + if (optEvent.isPresent()) { lastSynchronization = optEvent.get().getLastSynchronization(); event = optEvent.get(); - } else{ + } else { event = new SchemaSynchronizationEvent(); event.setSourceName(source.getId()); } - if(event.getErrorCount() != null && event.getErrorCount() > 3){ + if (event.getErrorCount() != null && event.getErrorCount() > 3) { LOGGER.warn("Synchronization with schema source with id {} at {} failed 3 times, skipping entry for now. Please check the synchronization source registry.", source.getId(), source.getBaseUrl()); continue; } @@ -113,7 +116,7 @@ public void performSynchronization(){ UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromHttpUrl(source.getBaseUrl()); - if(lastSynchronization != null){ + if (lastSynchronization != null) { uriBuilder = uriBuilder.queryParam("from", lastSynchronization); } @@ -121,85 +124,85 @@ public void performSynchronization(){ //@TODO switch to service identity, add bearer token optionally ResponseEntity response = restTemplate.exchange(uriBuilder.toUriString(), HttpMethod.GET, requestEntity, MetadataSchemaRecord[].class); - if(response.getStatusCodeValue() == 200){ + if (response.getStatusCode().value() == 200) { MetadataSchemaRecord[] receivedRecords = response.getBody(); - if(receivedRecords != null && receivedRecords.length > 0){ + if (receivedRecords != null && receivedRecords.length > 0) { LOGGER.trace("Performing update for {} schema(s).", receivedRecords.length); - for(MetadataSchemaRecord record : receivedRecords){ - LOGGER.trace("Checking record with schema with id {} in local schema registry.", record.getSchemaId()); + for (MetadataSchemaRecord schemaRecord : receivedRecords) { + LOGGER.trace("Checking record with schema with id {} in local schema registry.", schemaRecord.getSchemaId()); //obtain local schema // MetadataSchemaRecord foundRecord = null; try { - foundRecord = MetadataSchemaRecordUtil.getRecordById(schemaConfig, record.getSchemaId()); + foundRecord = MetadataSchemaRecordUtil.getRecordById(schemaConfig, schemaRecord.getSchemaId()); } catch (Exception ex) { // do nothing } Optional optRecord = Optional.ofNullable(foundRecord); - if(!optRecord.isPresent()){ - LOGGER.trace("New schema with id {} detected. Downloading remote schema document.", record.getSchemaId()); - createOrUpdateSchema(source.getBaseUrl(), localBaseUrl, record, event); - } else{ - LOGGER.trace("Existing schema with id {} detected. Downloading remote schema document.", record.getSchemaId()); + if (!optRecord.isPresent()) { + LOGGER.trace("New schema with id {} detected. Downloading remote schema document.", schemaRecord.getSchemaId()); + createOrUpdateSchema(source.getBaseUrl(), localBaseUrl, schemaRecord, event); + } else { + LOGGER.trace("Existing schema with id {} detected. Downloading remote schema document.", schemaRecord.getSchemaId()); MetadataSchemaRecord localRecord = optRecord.get(); - if(localRecord.getDoNotSync() == null || localRecord.getDoNotSync()){ + if (localRecord.getDoNotSync() == null || localRecord.getDoNotSync()) { LOGGER.trace("Synchronization of local record enabled. Comparing schema document hashes."); - if(!localRecord.getSchemaHash().equals(record.getSchemaHash())){ + if (!localRecord.getSchemaHash().equals(schemaRecord.getSchemaHash())) { //download remote schema - LOGGER.trace("Schema document hashes are NOT equal. Downloading schema document with id {} from {}.", record.getSchemaId(), source.getBaseUrl()); + LOGGER.trace("Schema document hashes are NOT equal. Downloading schema document with id {} from {}.", schemaRecord.getSchemaId(), source.getBaseUrl()); createOrUpdateSchema(source.getBaseUrl(), localBaseUrl, localRecord, event); - } else{ + } else { LOGGER.trace("Schema document hashes are equal. Skipping update."); updateSynchronizationEvent(event, true); - LOGGER.trace("Schema synchronization finished."); + LOGGER.trace(LOG_SYNCHRONIZATION_FINISHED); } - } else{ + } else { //skip LOGGER.trace("Synchronization for local record is disabled. Skipping synchronization of schema {}.", localRecord.getSchemaId()); updateSynchronizationEvent(event, true); - LOGGER.trace("Schema synchronization finished."); + LOGGER.trace(LOG_SYNCHRONIZATION_FINISHED); } } } - } else{ + } else { LOGGER.trace("No new or updated schemas received."); } - } else{ - LOGGER.error("Failed to obtain updates schemas from URL {}. Service returned status {}.", uriBuilder.toUriString(), response.getStatusCodeValue()); + } else { + LOGGER.error("Failed to obtain updates schemas from URL {}. Service returned status {}.", uriBuilder.toUriString(), response.getStatusCode().value()); } } } - private void createOrUpdateSchema(String baseUrl, String localBaseUrl, MetadataSchemaRecord record, SchemaSynchronizationEvent event){ + private void createOrUpdateSchema(String baseUrl, String localBaseUrl, MetadataSchemaRecord schemaRecord, SchemaSynchronizationEvent event) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); - int status = SimpleServiceClient.create(baseUrl).accept(MediaType.APPLICATION_OCTET_STREAM).withResourcePath(record.getSchemaId()).getResource(stream); - if(status == 200){ - try{ + int status = SimpleServiceClient.create(baseUrl).accept(MediaType.APPLICATION_OCTET_STREAM).withResourcePath(schemaRecord.getSchemaId()).getResource(stream); + if (status == 200) { + try { LOGGER.trace("Remote schema document successfully downloaded. Updating schema at local registry."); - HttpStatus createStatus = SimpleServiceClient.create(localBaseUrl).withFormParam("record", record).withFormParam("schema", new ByteArrayInputStream(stream.toByteArray())).postForm(); - if(HttpStatus.CREATED.equals(createStatus)){ - LOGGER.trace("New schema with id {} successfully updated. Updating lastSynchronization timestamp.", record.getSchemaId()); + HttpStatus createStatus = SimpleServiceClient.create(localBaseUrl).withFormParam("record", schemaRecord).withFormParam("schema", new ByteArrayInputStream(stream.toByteArray())).postForm(); + if (HttpStatus.CREATED.equals(createStatus)) { + LOGGER.trace("New schema with id {} successfully updated. Updating lastSynchronization timestamp.", schemaRecord.getSchemaId()); updateSynchronizationEvent(event, true); - LOGGER.trace("Schema synchronization finished."); - } else{ + LOGGER.trace(LOG_SYNCHRONIZATION_FINISHED); + } else { LOGGER.error("Failed to update schema at local registry, service returned {}. Updating errorCount.", createStatus); updateSynchronizationEvent(event, false); - LOGGER.trace("Schema synchronization finished."); + LOGGER.trace(LOG_SYNCHRONIZATION_FINISHED); } - } catch(IOException ex){ + } catch (IOException ex) { LOGGER.error("Failed to update schema record locally.", ex); updateSynchronizationEvent(event, false); - LOGGER.trace("Schema synchronization finished."); + LOGGER.trace(LOG_SYNCHRONIZATION_FINISHED); } } } - private void updateSynchronizationEvent(SchemaSynchronizationEvent event, boolean success){ + private void updateSynchronizationEvent(SchemaSynchronizationEvent event, boolean success) { event.setLastSynchronization(Instant.now()); - if(success){ + if (success) { event.setErrorCount((short) 0); - } else{ + } else { event.setErrorCount((short) (event.getErrorCount() + 1)); } LOGGER.trace("Writing updated synchronization event."); diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/ActuatorUtil.java b/src/main/java/edu/kit/datamanager/metastore2/util/ActuatorUtil.java index 6fabf988..2d38061c 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/ActuatorUtil.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/ActuatorUtil.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -28,50 +29,69 @@ import org.slf4j.LoggerFactory; /** - * Utility class for actuators collecting information details about - * local directory. + * Utility class for actuators collecting information details about local + * directory. */ public class ActuatorUtil { - /** + + /** * Logger */ private static final Logger LOG = LoggerFactory.getLogger(ActuatorUtil.class); + private ActuatorUtil() { + //Utility class + } + /** * Determine all details for given directory. + * * @param pathUrl URL of directory * @return Map with details. */ public static final Map testDirectory(URL pathUrl) { + Map properties = new HashMap<>(); + try { + Path path = Paths.get(pathUrl.toURI()); + properties = determineDetailsForPath(path); + } catch (URISyntaxException ex) { + LOG.error("Invalid base path uri of '" + pathUrl.toString() + "'.", ex); + } + return properties; + } + + /** + * Determine all details for given directory. + * + * @param path URL of directory + * @return Map with details. + */ + private static final Map determineDetailsForPath(Path path) { Map properties = new HashMap<>(); String totalSpace; String freeSpace; + Path probe = Paths.get(path.toString(), "probe.txt"); try { - Path path = Paths.get(pathUrl.toURI()); - Path probe = Paths.get(path.toString(), "probe.txt"); - try { - probe = Files.createFile(probe); - Files.write(probe, "Success".getBytes()); - File repoDir = path.toFile(); - double total = (double) repoDir.getTotalSpace(); - double free = (double) repoDir.getFreeSpace(); - totalSpace = String.format("%.2f GB", total / 1073741824); - freeSpace = String.format("%.2f GB (%.0f%%)", (double) repoDir.getFreeSpace() / 1073741824, free * 100.0 / total); - properties.put("Total space", totalSpace); - properties.put("Free space", freeSpace); + probe = Files.createFile(probe); + Files.write(probe, "Success".getBytes(StandardCharsets.UTF_8)); + File repoDir = path.toFile(); + double total = repoDir.getTotalSpace(); + double free = repoDir.getFreeSpace(); + totalSpace = String.format("%.2f GB", total / 1073741824); + freeSpace = String.format("%.2f GB (%.0f%%)", (double) repoDir.getFreeSpace() / 1073741824, free * 100.0 / total); + properties.put("Total space", totalSpace); + properties.put("Free space", freeSpace); - } catch (IOException ioe) { - LOG.error("Failed to check repository folder at '" + path.toString() + "'."); - } finally { - try { - Files.deleteIfExists(probe); - } catch (IOException ignored) { - } + } catch (IOException ioe) { + LOG.error("Failed to check repository folder at '" + path.toString() + "'."); + } finally { + try { + Files.deleteIfExists(probe); + } catch (IOException ignored) { + LOG.error("Can't delete file '{}'.", probe.toString()); } - } catch (URISyntaxException ex) { - LOG.error("Invalid base path uri of '" + pathUrl.toString() + "'.", ex); } return properties; } - + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/DownloadUtil.java b/src/main/java/edu/kit/datamanager/metastore2/util/DownloadUtil.java index b1a30129..5d3c376f 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/DownloadUtil.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/DownloadUtil.java @@ -53,12 +53,17 @@ public class DownloadUtil { private static final int MAX_LENGTH_OF_HEADER = 100; - private static final Pattern JSON_FIRST_BYTE = Pattern.compile("(\\R\\s)*\\s*\\{\\s*\"(.|\\s)*", Pattern.MULTILINE);//^\\s{\\s*\".*"); + private static final Pattern JSON_FIRST_BYTE = Pattern.compile("(\\R\\s)*\\s*\\{\\s*\"(.|\\s)*", Pattern.MULTILINE); private static final Pattern XML_FIRST_BYTE = Pattern.compile("((.|\\s)*<\\?xml[^<]*)?\\s*<\\s*(\\w+:)?\\w+(.|\\s)*", Pattern.MULTILINE); + DownloadUtil() { + // Utility class + } + /** * Downloads or copy the file behind the given URI and returns its path on - * local disc. You should delete or move the file to another location afterwards. + * local disc. You should delete or move the file to another location + * afterwards. * * @param resourceURL the given URI * @return the path to the created file. @@ -104,15 +109,13 @@ public static Path fixFileExtension(Path pathToFile) { Path returnFile = pathToFile; Path renamedFile = pathToFile; try { - if ((pathToFile != null) && (pathToFile.toFile().exists())) { + if ((pathToFile != null) && pathToFile.toFile().exists()) { String contentOfFile = FileUtils.readFileToString(pathToFile.toFile(), StandardCharsets.UTF_8); - String newExtension = guessFileExtension(contentOfFile.getBytes()); - if (newExtension != null) { - if (!pathToFile.toString().endsWith(newExtension)) { - renamedFile = Paths.get(pathToFile.toString() + newExtension); - FileUtils.moveFile(pathToFile.toFile(), renamedFile.toFile()); - returnFile = renamedFile; - } + String newExtension = guessFileExtension(contentOfFile.getBytes(StandardCharsets.UTF_8)); + if ((newExtension != null) && !pathToFile.toString().endsWith(newExtension)) { + renamedFile = Paths.get(pathToFile.toString() + newExtension); + FileUtils.moveFile(pathToFile.toFile(), renamedFile.toFile()); + returnFile = renamedFile; } } } catch (IOException ex) { @@ -132,8 +135,8 @@ public static Path fixFileExtension(Path pathToFile) { */ public static Path createTempFile(String prefix, String suffix) { Path tempFile = null; - prefix = ((prefix == null) || (prefix.trim().isEmpty())) ? DEFAULT_PREFIX : prefix; - suffix = ((suffix == null) || (suffix.trim().isEmpty())) ? DEFAULT_SUFFIX : suffix; + prefix = ((prefix == null) || prefix.trim().isEmpty()) ? DEFAULT_PREFIX : prefix; + suffix = ((suffix == null) || suffix.trim().isEmpty()) ? DEFAULT_SUFFIX : suffix; try { tempFile = Files.createTempFile(prefix, suffix); } catch (IllegalArgumentException | IOException ioe) { @@ -154,13 +157,12 @@ public static void removeFile(Path tempFile) { } catch (IOException ioe) { throw new CustomInternalServerError("Error removing file '" + tempFile.toString() + "'!"); } - return; } private static String guessFileExtension(byte[] schema) { // Cut schema to a maximum of MAX_LENGTH_OF_HEADER characters. int length = schema.length > MAX_LENGTH_OF_HEADER ? MAX_LENGTH_OF_HEADER : schema.length; - String schemaAsString = new String(schema, 0, length); + String schemaAsString = new String(schema, 0, length, StandardCharsets.UTF_8); LOGGER.trace("Guess type for '{}'", schemaAsString); Matcher m = JSON_FIRST_BYTE.matcher(schemaAsString); diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/JsonUtils.java b/src/main/java/edu/kit/datamanager/metastore2/util/JsonUtils.java index 7bf17ab0..264e4279 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/JsonUtils.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/JsonUtils.java @@ -21,12 +21,17 @@ import com.networknt.schema.JsonSchemaException; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion.VersionFlag; +import static com.networknt.schema.SpecVersion.VersionFlag.V201909; +import static com.networknt.schema.SpecVersion.VersionFlag.V6; +import static com.networknt.schema.SpecVersion.VersionFlag.V7; import com.networknt.schema.SpecVersionDetector; import com.networknt.schema.ValidationMessage; import edu.kit.datamanager.clients.SimpleServiceClient; import edu.kit.datamanager.metastore2.exception.JsonValidationException; import java.io.InputStream; import java.net.URI; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -57,12 +62,16 @@ public class JsonUtils { * Mapper for parsing json. */ private static final ObjectMapper mapper = new ObjectMapper(); + + JsonUtils() { + //Utility class + } /** * Validate JSON schema document based on detected JSON schema or version - * 2019-09 if no schema is defined. + * 2020-12 if no schema is defined. * - * @see http://json-schema.org/draft/2019-09/json-schema-core.html + * @see https://json-schema.org/draft/2020-12/json-schema-core.html * @param jsonSchemaStream schema document as string * @return true if schema is valid. */ @@ -74,9 +83,9 @@ public static boolean validateJsonSchemaDocument(InputStream jsonSchemaStream) t /** * Validate JSON schema document based on detected JSON schema or version - * 2019-09 if no schema is defined. + * 2020-12 if no schema is defined. * - * @see http://json-schema.org/draft/2019-09/json-schema-core.html + * @see https://json-schema.org/draft/2020-12/json-schema-core.html * @param jsonSchema schema document as string * @return true if schema is valid. */ @@ -89,7 +98,7 @@ public static boolean validateJsonSchemaDocument(String jsonSchema) throws JsonV /** * Validate JSON schema document based on JSON Schema. * - * @see http://json-schema.org/draft/2019-09/json-schema-core.html + * @see https://json-schema.org/draft/2020-12/json-schema-core.html * @see VersionFlag * @param jsonSchemaStream schema document as string * @param version use specific version @@ -104,7 +113,7 @@ public static boolean validateJsonSchemaDocument(InputStream jsonSchemaStream, V /** * Validate JSON schema document based on JSON Schema. * - * @see http://json-schema.org/draft/2019-09/json-schema-core.html + * @see https://json-schema.org/draft/2020-12/json-schema-core.html * @see VersionFlag * @param jsonSchema schema document as string * @param version use specific version @@ -115,8 +124,7 @@ public static boolean validateJsonSchemaDocument(String jsonSchema, VersionFlag try { // validate schema with meta schema. validateJson(jsonSchema, getSchema(version)); - JsonSchema schema = getJsonSchemaFromString(jsonSchema, version); - checkSchema(schema); + getJsonSchemaFromString(jsonSchema, version); } catch (Exception ex) { LOG.error("Unknown error", ex); String errorMessage = ex.getMessage(); @@ -183,7 +191,6 @@ public static boolean validateJson(String jsonDocument, String jsonSchema, Versi StringBuilder errorMessage = new StringBuilder(ERROR_VALIDATING_JSON_DOCUMENT); try { JsonSchema jsonSchemaFromString = getJsonSchemaFromString(jsonSchema, version); - checkSchema(jsonSchemaFromString); JsonNode jsonNode = getJsonNodeFromString(jsonDocument); Set validate = jsonSchemaFromString.validate(jsonNode); for (ValidationMessage message : validate) { @@ -211,7 +218,7 @@ public static boolean validateJson(String jsonDocument, String jsonSchema, Versi * @throws JsonValidationException if schema is not in correct format. */ private static VersionFlag determineSchemaVersion(String jsonSchema) throws JsonValidationException { - VersionFlag version = VersionFlag.V201909; + VersionFlag version = VersionFlag.V202012; try { JsonNode jsonNode = getJsonNodeFromString(jsonSchema); version = SpecVersionDetector.detect(jsonNode); @@ -235,7 +242,18 @@ private static VersionFlag determineSchemaVersion(String jsonSchema) throws Json */ protected static JsonSchema getJsonSchemaFromString(String schemaContent, VersionFlag version) throws Exception { JsonSchemaFactory factory = JsonSchemaFactory.getInstance(version); - return factory.getSchema(schemaContent); + JsonSchema schema = factory.getSchema(schemaContent); + checkSchema(schema); + Optional optionalVersionFound = SpecVersionDetector.detectOptionalVersion(schema.getSchemaNode()); + VersionFlag versionFound = version; + if (!optionalVersionFound.isEmpty()) { + versionFound = optionalVersionFound.get(); + } + if (!Objects.equals(version, versionFound)) { + LOG.error("Unknown MetaSchema or not matching: Expected: '{}' <-> Found: '{}'", version, versionFound); + throw new JsonSchemaException("Unknown MetaSchema or not matching: Expected: '" + version + "' <-> Found: '" + versionFound + "'"); + } + return schema; } /** @@ -267,12 +285,12 @@ private static String transformStreamToString(InputStream inputStream) throws Js } /** - * Test wether the schema has at least one validator. + * Test whether the schema has at least one validator. * * @param schema schema to check */ protected static void checkSchema(JsonSchema schema) { - if (schema.getValidators().isEmpty()) { + if (schema.getSchemaNode().isEmpty()) { throw new JsonValidationException(EMPTY_SCHEMA_DETECTED); } } @@ -290,19 +308,13 @@ protected static String getSchema(VersionFlag version) { try { switch (version) { case V4: - resourceUrl = new URI("http://json-schema.org/draft-04/schema"); - break; case V6: - resourceUrl = new URI("http://json-schema.org/draft-06/schema"); - break; case V7: - resourceUrl = new URI("http://json-schema.org/draft-07/schema"); - break; case V201909: - resourceUrl = new URI("http://json-schema.org/draft/2019-09/schema"); + case V202012: + resourceUrl = new URI(version.getId()); break; default: - resourceUrl = new URI("http://unknown.json.schema"); String message = String.format(UNKNOWN_JSON_SCHEMA + " '%s'", version); LOG.error(message); throw new JsonValidationException(message); diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataRecordUtil.java b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataRecordUtil.java index 0aaca712..1532ea25 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataRecordUtil.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataRecordUtil.java @@ -19,6 +19,7 @@ import edu.kit.datamanager.clients.SimpleServiceClient; import edu.kit.datamanager.entities.Identifier; import edu.kit.datamanager.entities.PERMISSION; +import edu.kit.datamanager.entities.RepoUserRole; import edu.kit.datamanager.exceptions.AccessForbiddenException; import edu.kit.datamanager.exceptions.BadArgumentException; import edu.kit.datamanager.exceptions.CustomInternalServerError; @@ -32,6 +33,7 @@ import edu.kit.datamanager.metastore2.domain.ResourceIdentifier; import edu.kit.datamanager.metastore2.domain.ResourceIdentifier.IdentifierType; import edu.kit.datamanager.metastore2.domain.SchemaRecord; +import edu.kit.datamanager.metastore2.web.impl.MetadataControllerImpl; import edu.kit.datamanager.repo.configuration.RepoBaseConfiguration; import edu.kit.datamanager.repo.domain.ContentInformation; import edu.kit.datamanager.repo.domain.DataResource; @@ -56,12 +58,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -70,6 +73,7 @@ import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.hateoas.server.mvc.WebMvcLinkBuilder; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.core.Authentication; @@ -93,19 +97,32 @@ public class MetadataRecordUtil { */ private static final Logger LOG = LoggerFactory.getLogger(MetadataRecordUtil.class); + private static final String LOG_SCHEMA_REGISTRY = "No external schema registries defined. Try to use internal one..."; + private static final String LOG_FETCH_SCHEMA = "Try to fetch schema from '{}'."; + private static final String PATH_SCHEMA = "schemas"; + private static final String LOG_ERROR_ACCESS = "Failed to access schema registry at '%s'. Proceeding with next registry."; + private static final String LOG_SCHEMA_RECORD = "Found schema record: '{}'"; + private static MetastoreConfiguration schemaConfig; - /** - * Encoding for strings/inputstreams. - */ - private static final String ENCODING = "UTF-8"; private static String guestToken = null; private static IDataRecordDao dataRecordDao; + MetadataRecordUtil() { + //Utility class + } + + /** + * Create a digital object from metadata record and metadata document. + * + * @param applicationProperties Configuration properties. + * @param recordDocument Metadata record. + * @param document Metadata document. + * @return Enriched metadata record. + */ public static MetadataRecord createMetadataRecord(MetastoreConfiguration applicationProperties, MultipartFile recordDocument, MultipartFile document) { - MetadataRecord result = null; - MetadataRecord record; + MetadataRecord metadataRecord; long nano1 = System.nanoTime() / 1000000; // Do some checks first. if (recordDocument == null || recordDocument.isEmpty() || document == null || document.isEmpty()) { @@ -114,7 +131,7 @@ public static MetadataRecord createMetadataRecord(MetastoreConfiguration applica throw new BadArgumentException(message); } try { - record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); + metadataRecord = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); } catch (IOException ex) { String message = "No valid metadata record provided. Returning HTTP BAD_REQUEST."; if (ex instanceof JsonParseException) { @@ -124,32 +141,32 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord throw new BadArgumentException(message); } - if (record.getRelatedResource() == null || record.getRelatedResource().getIdentifier() == null || record.getSchema() == null || record.getSchema().getIdentifier() == null) { + if (metadataRecord.getRelatedResource() == null || metadataRecord.getRelatedResource().getIdentifier() == null || metadataRecord.getSchema() == null || metadataRecord.getSchema().getIdentifier() == null) { String message = "Mandatory attributes relatedResource and/or schema not found in record. Returning HTTP BAD_REQUEST."; LOG.error(message); throw new BadArgumentException(message); } // Test for schema version - if (record.getSchemaVersion() == null) { + if (metadataRecord.getSchemaVersion() == null) { MetadataSchemaRecord currentSchemaRecord; try { - currentSchemaRecord = MetadataSchemaRecordUtil.getCurrentSchemaRecord(applicationProperties, record.getSchema()); + currentSchemaRecord = MetadataSchemaRecordUtil.getCurrentSchemaRecord(applicationProperties, metadataRecord.getSchema()); } catch (ResourceNotFoundException rnfe) { - throw new UnprocessableEntityException("Unknown schema ID '" + record.getSchema().getIdentifier() + "'!"); + throw new UnprocessableEntityException("Unknown schema ID '" + metadataRecord.getSchema().getIdentifier() + "'!"); } - record.setSchemaVersion(currentSchemaRecord.getSchemaVersion()); + metadataRecord.setSchemaVersion(currentSchemaRecord.getSchemaVersion()); } // validate document long nano2 = System.nanoTime() / 1000000; // validate schema document - validateMetadataDocument(applicationProperties, record, document); + validateMetadataDocument(applicationProperties, metadataRecord, document); // set internal parameters - record.setRecordVersion(1l); + metadataRecord.setRecordVersion(1l); long nano3 = System.nanoTime() / 1000000; // create record. - DataResource dataResource = migrateToDataResource(applicationProperties, record); + DataResource dataResource = migrateToDataResource(applicationProperties, metadataRecord); // add id as internal identifier if exists // Note: DataResourceUtils.createResource will ignore id of resource. // id will be set to alternate identifier if exists. @@ -164,7 +181,7 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord throw new BadArgumentException(message); } } catch (UnsupportedEncodingException ex) { - String message = "Error encoding id " + record.getSchemaId(); + String message = "Error encoding id " + metadataRecord.getSchemaId(); LOG.error(message); throw new CustomInternalServerError(message); } @@ -175,16 +192,14 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord DataResource createResource = DataResourceUtils.createResource(applicationProperties, dataResource); long nano5 = System.nanoTime() / 1000000; // store document - ContentInformation contentInformation = ContentDataUtils.addFile(applicationProperties, createResource, document, document.getOriginalFilename(), null, true, (t) -> { - return "somethingStupid"; - }); + ContentInformation contentInformation = ContentDataUtils.addFile(applicationProperties, createResource, document, document.getOriginalFilename(), null, true, t -> "somethingStupid"); long nano6 = System.nanoTime() / 1000000; // Create additional metadata record for faster access DataRecord dataRecord = new DataRecord(); dataRecord.setMetadataId(createResource.getId()); - dataRecord.setVersion(record.getRecordVersion()); - dataRecord.setSchemaId(record.getSchema().getIdentifier()); - dataRecord.setSchemaVersion(record.getSchemaVersion()); + dataRecord.setVersion(metadataRecord.getRecordVersion()); + dataRecord.setSchemaId(metadataRecord.getSchema().getIdentifier()); + dataRecord.setSchemaVersion(metadataRecord.getSchemaVersion()); dataRecord.setMetadataDocumentUri(contentInformation.getContentUri()); dataRecord.setDocumentHash(contentInformation.getHash()); dataRecord.setLastUpdate(dataResource.getLastUpdate()); @@ -195,15 +210,26 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord return migrateToMetadataRecord(applicationProperties, createResource, true); } + /** + * Update a digital object with given metadata record and/or metadata + * document. + * + * @param applicationProperties Configuration properties. + * @param resourceId Identifier of digital object. + * @param eTag ETag of the old digital object. + * @param recordDocument Metadata record. + * @param document Metadata document. + * @param supplier Function for updating record. + * @return Enriched metadata record. + */ public static MetadataRecord updateMetadataRecord(MetastoreConfiguration applicationProperties, String resourceId, String eTag, MultipartFile recordDocument, MultipartFile document, - Function supplier) { - MetadataRecord record = null; + UnaryOperator supplier) { + MetadataRecord metadataRecord = null; MetadataRecord existingRecord; - DataResource newResource; // Do some checks first. if ((recordDocument == null || recordDocument.isEmpty()) && (document == null || document.isEmpty())) { @@ -213,7 +239,7 @@ public static MetadataRecord updateMetadataRecord(MetastoreConfiguration applica } if (!(recordDocument == null || recordDocument.isEmpty())) { try { - record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); + metadataRecord = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); } catch (IOException ex) { String message = "Can't map record document to MetadataRecord"; if (ex instanceof JsonParseException) { @@ -228,53 +254,59 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord DataResource dataResource = applicationProperties.getDataResourceService().findById(resourceId); LOG.trace("Checking provided ETag."); ControllerUtils.checkEtag(eTag, dataResource); - if (record != null) { + if (metadataRecord != null) { existingRecord = migrateToMetadataRecord(applicationProperties, dataResource, false); - existingRecord = mergeRecords(existingRecord, record); + existingRecord = mergeRecords(existingRecord, metadataRecord); dataResource = migrateToDataResource(applicationProperties, existingRecord); } else { dataResource = DataResourceUtils.copyDataResource(dataResource); } - boolean noChanges = true; + boolean noChanges = false; if (document != null) { - record = migrateToMetadataRecord(applicationProperties, dataResource, false); - validateMetadataDocument(applicationProperties, record, document); + metadataRecord = migrateToMetadataRecord(applicationProperties, dataResource, false); + validateMetadataDocument(applicationProperties, metadataRecord, document); ContentInformation info; + String fileName = document.getOriginalFilename(); info = getContentInformationOfResource(applicationProperties, dataResource); - // Check for changes... - try { - byte[] currentFileContent; - File file = new File(URI.create(info.getContentUri())); - if (document.getSize() == Files.size(file.toPath())) { - currentFileContent = FileUtils.readFileToByteArray(file); - byte[] newFileContent = document.getBytes(); - for (int index = 0; index < currentFileContent.length; index++) { - if (currentFileContent[index] != newFileContent[index]) { - noChanges = false; - break; + if (info != null) { + fileName = info.getRelativePath(); + noChanges = true; + // Check for changes... + try { + byte[] currentFileContent; + File file = new File(URI.create(info.getContentUri())); + if (document.getSize() == Files.size(file.toPath())) { + currentFileContent = FileUtils.readFileToByteArray(file); + byte[] newFileContent = document.getBytes(); + for (int index = 0; index < currentFileContent.length; index++) { + if (currentFileContent[index] != newFileContent[index]) { + noChanges = false; + break; + } } + } else { + noChanges = false; } - } else { - noChanges = false; + } catch (IOException ex) { + LOG.error("Error reading current file!", ex); } - } catch (IOException ex) { - LOG.error("Error reading current file!", ex); } - if (noChanges == false) { + if (!noChanges) { // Everything seems to be fine update document and increment version LOG.trace("Updating schema document (and increment version)..."); String version = dataResource.getVersion(); if (version != null) { dataResource.setVersion(Long.toString(Long.parseLong(version) + 1l)); } - ContentDataUtils.addFile(applicationProperties, dataResource, document, info.getRelativePath(), null, true, supplier); + ContentDataUtils.addFile(applicationProperties, dataResource, document, fileName, null, true, supplier); } + } else { // validate if document is still valid due to changed record settings. - record = migrateToMetadataRecord(applicationProperties, dataResource, false); - URI metadataDocumentUri = URI.create(record.getMetadataDocumentUri()); + metadataRecord = migrateToMetadataRecord(applicationProperties, dataResource, false); + URI metadataDocumentUri = URI.create(metadataRecord.getMetadataDocumentUri()); Path metadataDocumentPath = Paths.get(metadataDocumentUri); if (!Files.exists(metadataDocumentPath) || !Files.isRegularFile(metadataDocumentPath) || !Files.isReadable(metadataDocumentPath)) { @@ -284,7 +316,7 @@ record = migrateToMetadataRecord(applicationProperties, dataResource, false); try { InputStream inputStream = Files.newInputStream(metadataDocumentPath); - SchemaRecord schemaRecord = MetadataSchemaRecordUtil.getSchemaRecord(record.getSchema(), record.getSchemaVersion()); + SchemaRecord schemaRecord = MetadataSchemaRecordUtil.getSchemaRecord(metadataRecord.getSchema(), metadataRecord.getSchemaVersion()); MetadataSchemaRecordUtil.validateMetadataDocument(applicationProperties, inputStream, schemaRecord); } catch (IOException ex) { LOG.error("Error validating file!", ex); @@ -302,15 +334,27 @@ record = migrateToMetadataRecord(applicationProperties, dataResource, false); return migrateToMetadataRecord(applicationProperties, dataResource, true); } + /** + * Delete a digital object with given identifier. + * + * @param applicationProperties Configuration properties. + * @param id Identifier of digital object. + * @param eTag ETag of the old digital object. + * @param supplier Function for updating record. + */ public static void deleteMetadataRecord(MetastoreConfiguration applicationProperties, String id, String eTag, - Function supplier) { + UnaryOperator supplier) { DataResourceUtils.deleteResource(applicationProperties, id, eTag, supplier); - Optional dataRecord = dataRecordDao.findTopByMetadataIdOrderByVersionDesc(id); - while (dataRecord.isPresent()) { - dataRecordDao.delete(dataRecord.get()); - dataRecord = dataRecordDao.findTopByMetadataIdOrderByVersionDesc(id); + try { + DataResourceUtils.getResourceByIdentifierOrRedirect(applicationProperties, id, null, supplier); + } catch (ResourceNotFoundException rnfe) { + Optional dataRecord = dataRecordDao.findTopByMetadataIdOrderByVersionDesc(id); + while (dataRecord.isPresent()) { + dataRecordDao.delete(dataRecord.get()); + dataRecord = dataRecordDao.findTopByMetadataIdOrderByVersionDesc(id); + } } } @@ -358,12 +402,14 @@ public static DataResource migrateToDataResource(RepoBaseConfiguration applicati MetadataSchemaRecordUtil.checkAlternateIdentifier(identifiers, identifier.getIdentifier(), Identifier.IDENTIFIER_TYPE.valueOf(identifier.getIdentifierType().name())); } else { LOG.trace("Remove existing identifiers (others than URL)..."); + Set removeItems = new HashSet<>(); for (Identifier item : identifiers) { if (item.getIdentifierType() != Identifier.IDENTIFIER_TYPE.URL) { LOG.trace("... {}, {}", item.getValue(), item.getIdentifierType()); - identifiers.remove(item); + removeItems.add(item); } } + identifiers.removeAll(removeItems); } boolean relationFound = false; boolean schemaIdFound = false; @@ -434,9 +480,7 @@ public static MetadataRecord migrateToMetadataRecord(RepoBaseConfiguration appli metadataRecord.setLastUpdate(dataResource.getLastUpdate()); } - Iterator iterator = dataResource.getAlternateIdentifiers().iterator(); - while (iterator.hasNext()) { - Identifier identifier = iterator.next(); + for (Identifier identifier : dataResource.getAlternateIdentifiers()) { if (identifier.getIdentifierType() != Identifier.IDENTIFIER_TYPE.URL) { if (identifier.getIdentifierType() != Identifier.IDENTIFIER_TYPE.INTERNAL) { ResourceIdentifier resourceIdentifier = ResourceIdentifier.factoryResourceIdentifier(identifier.getValue(), ResourceIdentifier.IdentifierType.valueOf(identifier.getIdentifierType().getValue())); @@ -460,8 +504,8 @@ public static MetadataRecord migrateToMetadataRecord(RepoBaseConfiguration appli if (relatedIds.getRelationType() == RelatedIdentifier.RELATION_TYPES.IS_METADATA_FOR) { ResourceIdentifier resourceIdentifier = ResourceIdentifier.factoryInternalResourceIdentifier(relatedIds.getValue()); if (relatedIds.getIdentifierType() != null) { - resourceIdentifier = ResourceIdentifier.factoryResourceIdentifier(relatedIds.getValue(), IdentifierType.valueOf(relatedIds.getIdentifierType().name())); - } + resourceIdentifier = ResourceIdentifier.factoryResourceIdentifier(relatedIds.getValue(), IdentifierType.valueOf(relatedIds.getIdentifierType().name())); + } LOG.trace("Set relation to '{}'", resourceIdentifier); metadataRecord.setRelatedResource(resourceIdentifier); } @@ -470,7 +514,7 @@ public static MetadataRecord migrateToMetadataRecord(RepoBaseConfiguration appli metadataRecord.setSchema(resourceIdentifier); if (resourceIdentifier.getIdentifierType().equals(IdentifierType.URL)) { //Try to fetch version from URL (only works with URLs including the version as query parameter. - Matcher matcher = Pattern.compile(".*[&?]version=([0-9]*).*").matcher(resourceIdentifier.getIdentifier()); + Matcher matcher = Pattern.compile(".*[&?]version=(\\d*).*").matcher(resourceIdentifier.getIdentifier()); while (matcher.find()) { metadataRecord.setSchemaVersion(Long.parseLong(matcher.group(1))); } @@ -480,6 +524,11 @@ public static MetadataRecord migrateToMetadataRecord(RepoBaseConfiguration appli LOG.trace("Set schema to '{}'", resourceIdentifier); } } + if (metadataRecord.getSchema() == null) { + String message = "Missing schema identifier for metadata document. Not a valid metadata document ID. Returning HTTP BAD_REQUEST."; + LOG.error(message); + throw new BadArgumentException(message); + } DataRecord dataRecord = null; long nano2 = System.nanoTime() / 1000000; Optional dataRecordResult = dataRecordDao.findByMetadataIdAndVersion(dataResource.getId(), recordVersion); @@ -559,16 +608,16 @@ public static MetadataSchemaRecord getCurrentInternalSchemaRecord(MetastoreConfi MetadataSchemaRecord returnValue = null; boolean success = false; StringBuilder errorMessage = new StringBuilder(); - if (metastoreProperties.getSchemaRegistries().length == 0) { - LOG.trace("No external schema registries defined. Try to use internal one..."); + if (metastoreProperties.getSchemaRegistries().size() == 0) { + LOG.trace(LOG_SCHEMA_REGISTRY); returnValue = MetadataSchemaRecordUtil.getRecordById(metastoreProperties, schemaId); success = true; } else { for (String schemaRegistry : metastoreProperties.getSchemaRegistries()) { - LOG.trace("Try to fetch schema from '{}'.", schemaRegistry); + LOG.trace(LOG_FETCH_SCHEMA, schemaRegistry); URI schemaRegistryUri = URI.create(schemaRegistry); - UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), "schemas", schemaId); + UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), PATH_SCHEMA, schemaId); URI finalUri = builder.build().toUri(); @@ -581,7 +630,7 @@ public static MetadataSchemaRecord getCurrentInternalSchemaRecord(MetastoreConfi LOG.error(message, ce); errorMessage.append(message).append("\n"); } catch (RestClientException ex) { - String message = "Failed to access schema registry at '" + schemaRegistry + "'. Proceeding with next registry."; + String message = String.format(LOG_ERROR_ACCESS, schemaRegistry); LOG.error(message, ex); errorMessage.append(message).append("\n"); } @@ -590,7 +639,7 @@ public static MetadataSchemaRecord getCurrentInternalSchemaRecord(MetastoreConfi if (!success) { throw new UnprocessableEntityException(errorMessage.toString()); } - LOG.trace("Found schema record: '{}'", returnValue); + LOG.trace(LOG_SCHEMA_RECORD, returnValue); return returnValue; } @@ -609,16 +658,16 @@ public static MetadataSchemaRecord getInternalSchemaRecord(MetastoreConfiguratio boolean success = false; StringBuilder errorMessage = new StringBuilder(); LOG.trace("Get internal schema record for id '{}'.", schemaId); - if (metastoreProperties.getSchemaRegistries().length == 0) { - LOG.trace("No external schema registries defined. Try to use internal one..."); + if (metastoreProperties.getSchemaRegistries().size() == 0) { + LOG.trace(LOG_SCHEMA_REGISTRY); returnValue = MetadataSchemaRecordUtil.getRecordByIdAndVersion(metastoreProperties, schemaId, version); success = true; } else { for (String schemaRegistry : metastoreProperties.getSchemaRegistries()) { - LOG.trace("Try to fetch schema from '{}'.", schemaRegistry); + LOG.trace(LOG_FETCH_SCHEMA, schemaRegistry); URI schemaRegistryUri = URI.create(schemaRegistry); - UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), "schemas", schemaId).queryParam("version", version); + UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), PATH_SCHEMA, schemaId).queryParam("version", version); URI finalUri = builder.build().toUri(); @@ -631,7 +680,7 @@ public static MetadataSchemaRecord getInternalSchemaRecord(MetastoreConfiguratio LOG.error(message, ce); errorMessage.append(message).append("\n"); } catch (RestClientException ex) { - String message = "Failed to access schema registry at '" + schemaRegistry + "'. Proceeding with next registry."; + String message = String.format(LOG_ERROR_ACCESS, schemaRegistry); LOG.error(message, ex); errorMessage.append(message).append("\n"); } @@ -640,7 +689,7 @@ public static MetadataSchemaRecord getInternalSchemaRecord(MetastoreConfiguratio if (!success) { throw new UnprocessableEntityException(errorMessage.toString()); } - LOG.trace("Found schema record: '{}'", returnValue); + LOG.trace(LOG_SCHEMA_RECORD, returnValue); return returnValue; } @@ -667,14 +716,13 @@ private static RelatedIdentifier updateRelatedIdentifierForSchema(RelatedIdentif * Validate metadata document with given schema. * * @param metastoreProperties Configuration for accessing services - * @param record metadata of the document. + * @param metadataRecord metadata of the document. * @param document document - * @throws Exception In case of any error or invalid document. */ private static void validateMetadataDocument(MetastoreConfiguration metastoreProperties, - MetadataRecord record, + MetadataRecord metadataRecord, MultipartFile document) { - LOG.trace("validateMetadataDocument {},{}, {}", metastoreProperties, record, document); + LOG.trace("validateMetadataDocument {},{}, {}", metastoreProperties, metadataRecord, document); if (document == null || document.isEmpty()) { String message = "Missing metadata document in body. Returning HTTP BAD_REQUEST."; LOG.error(message); @@ -682,11 +730,11 @@ private static void validateMetadataDocument(MetastoreConfiguration metastorePro } boolean validationSuccess = false; StringBuilder errorMessage = new StringBuilder(); - if (metastoreProperties.getSchemaRegistries().length == 0 || record.getSchema().getIdentifierType() != IdentifierType.INTERNAL) { - LOG.trace("No external schema registries defined. Try to use internal one..."); + if (metastoreProperties.getSchemaRegistries().isEmpty() || metadataRecord.getSchema().getIdentifierType() != IdentifierType.INTERNAL) { + LOG.trace(LOG_SCHEMA_REGISTRY); if (schemaConfig != null) { try { - MetadataSchemaRecordUtil.validateMetadataDocument(schemaConfig, document, record.getSchema(), record.getSchemaVersion()); + MetadataSchemaRecordUtil.validateMetadataDocument(schemaConfig, document, metadataRecord.getSchema(), metadataRecord.getSchemaVersion()); validationSuccess = true; } catch (Exception ex) { String message = "Error validating document!"; @@ -698,9 +746,9 @@ private static void validateMetadataDocument(MetastoreConfiguration metastorePro } } else { for (String schemaRegistry : metastoreProperties.getSchemaRegistries()) { - LOG.trace("Try to fetch schema from '{}'.", schemaRegistry); + LOG.trace(LOG_FETCH_SCHEMA, schemaRegistry); URI schemaRegistryUri = URI.create(schemaRegistry); - UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), "schemas", record.getSchema().getIdentifier(), "validate").queryParam("version", record.getSchemaVersion()); + UriComponentsBuilder builder = UriComponentsBuilder.newInstance().scheme(schemaRegistryUri.getScheme()).host(schemaRegistryUri.getHost()).port(schemaRegistryUri.getPort()).pathSegment(schemaRegistryUri.getPath(), PATH_SCHEMA, metadataRecord.getSchema().getIdentifier(), "validate").queryParam("version", metadataRecord.getSchemaVersion()); URI finalUri = builder.build().toUri(); @@ -708,17 +756,17 @@ private static void validateMetadataDocument(MetastoreConfiguration metastorePro HttpStatus status = SimpleServiceClient.create(finalUri.toString()).withBearerToken(guestToken).accept(MetadataSchemaRecord.METADATA_SCHEMA_RECORD_MEDIA_TYPE).withFormParam("document", document.getInputStream()).postForm(MediaType.MULTIPART_FORM_DATA); if (Objects.equals(HttpStatus.NO_CONTENT, status)) { - LOG.trace("Successfully validated document against schema {} in registry {}.", record.getSchema().getIdentifier(), schemaRegistry); + LOG.trace("Successfully validated document against schema {} in registry {}.", metadataRecord.getSchema().getIdentifier(), schemaRegistry); validationSuccess = true; break; } } catch (HttpClientErrorException ce) { //not valid - String message = "Failed to validate metadata document against schema " + record.getSchema().getIdentifier() + " at '" + schemaRegistry + "' with status " + ce.getStatusCode() + "."; + String message = "Failed to validate metadata document against schema " + metadataRecord.getSchema().getIdentifier() + " at '" + schemaRegistry + "' with status " + ce.getStatusCode() + "."; LOG.error(message, ce); errorMessage.append(message).append("\n"); } catch (IOException | RestClientException ex) { - String message = "Failed to access schema registry at '" + schemaRegistry + "'. Proceeding with next registry."; + String message = String.format(LOG_ERROR_ACCESS, schemaRegistry); LOG.error(message, ex); errorMessage.append(message).append("\n"); } @@ -727,8 +775,6 @@ private static void validateMetadataDocument(MetastoreConfiguration metastorePro if (!validationSuccess) { throw new UnprocessableEntityException(errorMessage.toString()); } - - return; } public static MetadataRecord getRecordByIdAndVersion(MetastoreConfiguration metastoreProperties, @@ -745,9 +791,16 @@ public static MetadataRecord getRecordByIdAndVersion(MetastoreConfiguration meta String recordId, Long version, boolean supportEtag) throws ResourceNotFoundException { //if security enabled, check permission -> if not matching, return HTTP UNAUTHORIZED or FORBIDDEN long nano = System.nanoTime() / 1000000; + long nano2; MetadataRecord result = null; - Page dataResource = metastoreProperties.getDataResourceService().findAllVersions(recordId, null); - long nano2 = System.nanoTime() / 1000000; + Page dataResource; + try { + dataResource = metastoreProperties.getDataResourceService().findAllVersions(recordId, null); + } catch (ResourceNotFoundException ex) { + ex.setDetail("Metadata document with ID '" + recordId + "' doesn't exist!"); + throw ex; + } + nano2 = System.nanoTime() / 1000000; Stream stream = dataResource.get(); if (version != null) { stream = stream.filter(resource -> Long.parseLong(resource.getVersion()) == version); @@ -756,9 +809,9 @@ public static MetadataRecord getRecordByIdAndVersion(MetastoreConfiguration meta if (findFirst.isPresent()) { result = migrateToMetadataRecord(metastoreProperties, findFirst.get(), supportEtag); } else { - String message = String.format("ID '%s' or version '%d' doesn't exist!", recordId, version.longValue()); + String message = String.format("Version '%d' of ID '%s' doesn't exist!", version, recordId); LOG.error(message); - throw new BadArgumentException(message); + throw new ResourceNotFoundException(message); } long nano3 = System.nanoTime() / 1000000; LOG.info("getRecordByIdAndVersion {}, {}, {}", nano, (nano2 - nano), (nano3 - nano)); @@ -773,9 +826,9 @@ public static Path getMetadataDocumentByIdAndVersion(MetastoreConfiguration meta public static Path getMetadataDocumentByIdAndVersion(MetastoreConfiguration metastoreProperties, String recordId, Long version) throws ResourceNotFoundException { LOG.trace("Obtaining metadata record with id {} and version {}.", recordId, version); - MetadataRecord record = getRecordByIdAndVersion(metastoreProperties, recordId, version); + MetadataRecord metadataRecord = getRecordByIdAndVersion(metastoreProperties, recordId, version); - URI metadataDocumentUri = URI.create(record.getMetadataDocumentUri()); + URI metadataDocumentUri = URI.create(metadataRecord.getMetadataDocumentUri()); Path metadataDocumentPath = Paths.get(metadataDocumentUri); if (!Files.exists(metadataDocumentPath) || !Files.isRegularFile(metadataDocumentPath) || !Files.isReadable(metadataDocumentPath)) { @@ -785,55 +838,89 @@ public static Path getMetadataDocumentByIdAndVersion(MetastoreConfiguration meta return metadataDocumentPath; } + /** + * Merge new metadata record in the existing one. + * + * @param managed Existing metadata record. + * @param provided New metadata record. + * @return Merged record + */ public static MetadataRecord mergeRecords(MetadataRecord managed, MetadataRecord provided) { if (provided != null && managed != null) { //update pid - if (provided.getPid() != null) { - if (!provided.getPid().equals(managed.getPid())) { - LOG.trace("Updating record pid from {} to {}.", managed.getPid(), provided.getPid()); - managed.setPid(provided.getPid()); - } - } + managed.setPid(mergeEntry("Update record->pid", managed.getPid(), provided.getPid())); + //update acl - if (!provided.getAcl().isEmpty()) { - if (!provided.getAcl().equals(managed.getAcl())) { - // check for special access rights - // - only administrators are allowed to change ACL - if (checkAccessRights(managed.getAcl())) { - // - at least principal has to remain as ADMIN - if (checkAccessRights(provided.getAcl())) { - LOG.trace("Updating record acl from {} to {}.", managed.getAcl(), provided.getAcl()); - managed.setAcl(provided.getAcl()); - } - } - } - } + managed.setAcl(mergeAcl(managed.getAcl(), provided.getAcl())); //update getRelatedResource - if (provided.getRelatedResource() != null) { - if (!provided.getRelatedResource().equals(managed.getRelatedResource())) { - LOG.trace("Updating related resource from {} to {}.", managed.getRelatedResource(), provided.getRelatedResource()); - managed.setRelatedResource(provided.getRelatedResource()); - } - } + managed.setRelatedResource(mergeEntry("Updating record->relatedResource", managed.getRelatedResource(), provided.getRelatedResource())); //update schemaId - if (provided.getSchema() != null) { - if (!provided.getSchema().equals(managed.getSchema())) { - LOG.trace("Updating record schema from {} to {}.", managed.getSchema(), provided.getSchema()); - managed.setSchema(provided.getSchema()); - } - } + managed.setSchema(mergeEntry("Updating record->schema", managed.getSchema(), provided.getSchema())); + //update schemaVersion - if (provided.getSchemaVersion() != null) { - if (!provided.getSchemaVersion().equals(managed.getSchemaVersion())) { - LOG.trace("Updating record schemaVersion from {} to {}.", managed.getSchemaVersion(), provided.getSchemaVersion()); - managed.setSchemaVersion(provided.getSchemaVersion()); - } - } + managed.setSchemaVersion(mergeEntry("Updating record->schemaVersion", managed.getSchemaVersion(), provided.getSchemaVersion())); } else { managed = (managed != null) ? managed : provided; } -// LOG.trace("Setting lastUpdate to now()."); -// managed.setLastUpdate(Instant.now()); + return managed; + } + + /** + * Check validity of acl list and then merge new acl list in the existing one. + * + * @param managed Existing metadata record. + * @param provided New metadata record. + * @return Merged list + */ + public static Set mergeAcl(Set managed, Set provided) { + // Check for null parameters (which shouldn't happen) + managed = (managed == null) ? new HashSet<>() : managed; + provided = (provided == null) ? new HashSet<>() : provided; + if (!provided.isEmpty()) { + if (!provided.equals(managed)) { + // check for special access rights + // - only administrators are allowed to change ACL + checkAccessRights(managed, true); + // - at least principal has to remain as ADMIN + checkAccessRights(provided, false); + LOG.trace("Updating record acl from {} to {}.", managed, provided); + managed = provided; + } else { + LOG.trace("Provided ACL is still the same -> Continue using old one."); + } + } else { + LOG.trace("Provided ACL is empty -> Continue using old one."); + } + return managed; + } + + /** + * Set new value for existing one. + * + * @param description For logging purposes only + * @param managed Existing value. + * @param provided New value. + * @return Merged record + */ + public static T mergeEntry(String description, T managed, T provided) { + return mergeEntry(description, managed, provided, false); + } + + /** + * Set new value for existing one. + * + * @param description For logging purposes only + * @param managed Existing value. + * @param provided New value. + * @param overwriteWithNull Allows also deletion of a value. + * @return Merged record + */ + public static T mergeEntry(String description, T managed, T provided, boolean overwriteWithNull) { + if ((provided != null && !provided.equals(managed)) + || overwriteWithNull) { + LOG.trace(description + " from '{}' to '{}'", managed, provided); + managed = provided; + } return managed; } @@ -852,6 +939,8 @@ public static void setToken(String bearerToken) { } /** + * Set schema config. + * * @param aSchemaConfig the schemaConfig to set */ public static void setSchemaConfig(MetastoreConfiguration aSchemaConfig) { @@ -859,6 +948,8 @@ public static void setSchemaConfig(MetastoreConfiguration aSchemaConfig) { } /** + * Set DAO for data record. + * * @param aDataRecordDao the dataRecordDao to set */ public static void setDataRecordDao(IDataRecordDao aDataRecordDao) { @@ -866,7 +957,7 @@ public static void setDataRecordDao(IDataRecordDao aDataRecordDao) { } private static void saveNewDataRecord(MetadataRecord result) { - DataRecord dataRecord = null; + DataRecord dataRecord; // Create shortcut for access. LOG.trace("Save new data record!"); @@ -902,13 +993,27 @@ private static void saveNewDataRecord(DataRecord dataRecord) { } } - public static boolean checkAccessRights(Set aclEntries) { + /** + * Checks if current user is allowed to access with given AclEntries. + * + * @param aclEntries AclEntries of resource. + * @param currentAcl Check current ACL (true) or new one (false). + * + * @return Allowed (true) or not. + */ + public static boolean checkAccessRights(Set aclEntries, boolean currentAcl) { boolean isAllowed = false; + String errorMessage1 = "Error invalid ACL! Reason: Only ADMINISTRATORS are allowed to change ACL entries."; + String errorMessage2 = "Error invalid ACL! Reason: You are not allowed to revoke your own administrator rights."; Authentication authentication = AuthenticationHelper.getAuthentication(); List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); for (GrantedAuthority authority : authentication.getAuthorities()) { authorizationIdentities.add(authority.getAuthority()); } + if (authorizationIdentities.contains(RepoUserRole.ADMINISTRATOR.getValue())) { + //ROLE_ADMINISTRATOR detected -> no further permission check necessary. + return true; + } if (LOG.isTraceEnabled()) { LOG.trace("Check access rights for changing ACL list!"); for (String authority : authorizationIdentities) { @@ -919,21 +1024,31 @@ public static boolean checkAccessRights(Set aclEntries) { Iterator iterator = aclEntries.iterator(); while (iterator.hasNext()) { AclEntry aclEntry = iterator.next(); - LOG.trace("'{}' has ’{}' rights!", aclEntry.getSid(), aclEntry.getPermission()); - if (aclEntry.getPermission().atLeast(PERMISSION.ADMINISTRATE)) { - if (authorizationIdentities.contains(aclEntry.getSid())) { - isAllowed = true; - LOG.trace("'{}' has ’{}' rights!", aclEntry.getSid(), PERMISSION.ADMINISTRATE); - break; - } + LOG.trace("'{}' has ’{}' rights!", aclEntry.getSid(), aclEntry.getPermission()); + if (aclEntry.getPermission().atLeast(PERMISSION.ADMINISTRATE) + && authorizationIdentities.contains(aclEntry.getSid())) { + isAllowed = true; + LOG.trace("Confirm permission for updating ACL: '{}' has ’{}' rights!", aclEntry.getSid(), PERMISSION.ADMINISTRATE); + break; } } if (!isAllowed) { - LOG.warn("Only ADMINISTRATORS are allowed to change ACL entries"); + String errorMessage = currentAcl ? errorMessage1 : errorMessage2; + LOG.warn(errorMessage); if (schemaConfig.isAuthEnabled()) { - throw new AccessForbiddenException("Error wrong ACL!"); + if (currentAcl) { + throw new AccessForbiddenException(errorMessage1); + } else { + throw new BadArgumentException(errorMessage2); + } } } return isAllowed; } + + public static final void fixMetadataDocumentUri(MetadataRecord metadataRecord) { + String metadataDocumentUri = metadataRecord.getMetadataDocumentUri(); + metadataRecord.setMetadataDocumentUri(WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(MetadataControllerImpl.class).getMetadataDocumentById(metadataRecord.getId(), metadataRecord.getRecordVersion(), null, null)).toUri().toString()); + LOG.trace("Fix metadata document Uri '{}' -> '{}'", metadataDocumentUri, metadataRecord.getMetadataDocumentUri()); + } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java index 35dd286f..5b27d854 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java @@ -16,16 +16,17 @@ package edu.kit.datamanager.metastore2.util; import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.ObjectMapper; import edu.kit.datamanager.entities.Identifier; import edu.kit.datamanager.exceptions.BadArgumentException; import edu.kit.datamanager.exceptions.CustomInternalServerError; import edu.kit.datamanager.exceptions.ResourceNotFoundException; import edu.kit.datamanager.exceptions.UnprocessableEntityException; import edu.kit.datamanager.metastore2.configuration.MetastoreConfiguration; +import edu.kit.datamanager.metastore2.dao.IDataRecordDao; import edu.kit.datamanager.metastore2.dao.IMetadataFormatDao; import edu.kit.datamanager.metastore2.dao.ISchemaRecordDao; import edu.kit.datamanager.metastore2.dao.IUrl2PathDao; +import edu.kit.datamanager.metastore2.domain.DataRecord; import edu.kit.datamanager.metastore2.domain.MetadataRecord; import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord.SCHEMA_TYPE; @@ -34,7 +35,10 @@ import edu.kit.datamanager.metastore2.domain.SchemaRecord; import edu.kit.datamanager.metastore2.domain.Url2Path; import edu.kit.datamanager.metastore2.domain.oaipmh.MetadataFormat; +import static edu.kit.datamanager.metastore2.util.MetadataRecordUtil.mergeAcl; +import static edu.kit.datamanager.metastore2.util.MetadataRecordUtil.mergeEntry; import edu.kit.datamanager.metastore2.validation.IValidator; +import edu.kit.datamanager.metastore2.web.impl.SchemaRegistryControllerImpl; import edu.kit.datamanager.repo.configuration.RepoBaseConfiguration; import edu.kit.datamanager.repo.domain.ContentInformation; import edu.kit.datamanager.repo.domain.DataResource; @@ -66,15 +70,18 @@ import java.util.Optional; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.hateoas.server.mvc.WebMvcLinkBuilder; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; /** * Utility class for handling json documents @@ -85,14 +92,8 @@ public class MetadataSchemaRecordUtil { * Logger for messages. */ private static final Logger LOG = LoggerFactory.getLogger(MetadataSchemaRecordUtil.class); - /** - * Encoding for strings/inputstreams. - */ - private static final String ENCODING = "UTF-8"; - /** - * Mapper for parsing json. - */ - private static ObjectMapper mapper = new ObjectMapper(); + private static final String LOG_ERROR_READ_METADATA_DOCUMENT = "Failed to read metadata document from input stream."; + private static final String LOG_SEPARATOR = "-----------------------------------------"; private static ISchemaRecordDao schemaRecordDao; @@ -100,12 +101,26 @@ public class MetadataSchemaRecordUtil { private static IUrl2PathDao url2PathDao; + private static IDataRecordDao dataRecordDao; + + MetadataSchemaRecordUtil() { + //Utility class + } + + /** + * Create/Ingest an instance of MetadataSchemaRecord. + * + * @param applicationProperties Settings of repository. + * @param recordDocument Record of the schema. + * @param document Schema document. + * @param getSchemaDocumentById Method for creating access URL. + * @return Record of registered schema document. + */ public static MetadataSchemaRecord createMetadataSchemaRecord(MetastoreConfiguration applicationProperties, MultipartFile recordDocument, MultipartFile document, BiFunction getSchemaDocumentById) { - MetadataSchemaRecord result = null; - MetadataSchemaRecord record; + MetadataSchemaRecord metadataRecord; // Do some checks first. if (recordDocument == null || recordDocument.isEmpty() || document == null || document.isEmpty()) { @@ -114,7 +129,7 @@ public static MetadataSchemaRecord createMetadataSchemaRecord(MetastoreConfigura throw new BadArgumentException(message); } try { - record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchemaRecord.class); + metadataRecord = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchemaRecord.class); } catch (IOException ex) { String message = "No valid metadata record provided. Returning HTTP BAD_REQUEST."; if (ex instanceof JsonParseException) { @@ -125,61 +140,59 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema throw new BadArgumentException(message); } - if (record.getSchemaId() == null) { + if (metadataRecord.getSchemaId() == null) { String message = "Mandatory attributes schemaId not found in record. Returning HTTP BAD_REQUEST."; LOG.error(message); throw new BadArgumentException(message); } else { try { - String value = URLEncoder.encode(record.getSchemaId(), StandardCharsets.UTF_8.toString()); - if (!value.equals(record.getSchemaId())) { + String value = URLEncoder.encode(metadataRecord.getSchemaId(), StandardCharsets.UTF_8.toString()); + if (!value.equals(metadataRecord.getSchemaId())) { String message = "Not a valid schema id! Encoded: " + value; LOG.error(message); throw new BadArgumentException(message); } } catch (UnsupportedEncodingException ex) { - String message = "Error encoding schemaId " + record.getSchemaId(); + String message = "Error encoding schemaId " + metadataRecord.getSchemaId(); LOG.error(message); throw new CustomInternalServerError(message); } } // Create schema record SchemaRecord schemaRecord = new SchemaRecord(); - schemaRecord.setSchemaId(record.getSchemaId()); - schemaRecord.setType(record.getType()); + schemaRecord.setSchemaId(metadataRecord.getSchemaId()); + schemaRecord.setType(metadataRecord.getType()); // End of parameter checks // validate schema document / determine type if not given validateMetadataSchemaDocument(applicationProperties, schemaRecord, document); // set internal parameters - record.setType(schemaRecord.getType()); - if (record.getMimeType() == null) { + metadataRecord.setType(schemaRecord.getType()); + if (metadataRecord.getMimeType() == null) { LOG.trace("No mimetype set! Try to determine..."); if (document.getContentType() != null) { LOG.trace("Set mimetype determined from document: '{}'", document.getContentType()); - record.setMimeType(document.getContentType()); + metadataRecord.setMimeType(document.getContentType()); } else { - LOG.trace("Set mimetype according to type '{}'.", record.getType()); - switch (record.getType()) { + LOG.trace("Set mimetype according to type '{}'.", metadataRecord.getType()); + switch (metadataRecord.getType()) { case JSON: - record.setMimeType(MediaType.APPLICATION_JSON_VALUE); + metadataRecord.setMimeType(MediaType.APPLICATION_JSON_VALUE); break; case XML: - record.setMimeType(MediaType.APPLICATION_XML_VALUE); + metadataRecord.setMimeType(MediaType.APPLICATION_XML_VALUE); break; default: - throw new BadArgumentException("Please provide mimetype for type '" + record.getType() + "'"); + throw new BadArgumentException("Please provide mimetype for type '" + metadataRecord.getType() + "'"); } } } - record.setSchemaVersion(1l); + metadataRecord.setSchemaVersion(1l); // create record. - DataResource dataResource = migrateToDataResource(applicationProperties, record); + DataResource dataResource = migrateToDataResource(applicationProperties, metadataRecord); DataResource createResource = DataResourceUtils.createResource(applicationProperties, dataResource); // store document - ContentInformation contentInformation = ContentDataUtils.addFile(applicationProperties, createResource, document, document.getOriginalFilename(), null, true, (t) -> { - return "somethingStupid"; - }); + ContentInformation contentInformation = ContentDataUtils.addFile(applicationProperties, createResource, document, document.getOriginalFilename(), null, true, t -> "somethingStupid"); schemaRecord.setVersion(applicationProperties.getAuditService().getCurrentVersion(dataResource.getId())); schemaRecord.setSchemaDocumentUri(contentInformation.getContentUri()); schemaRecord.setDocumentHash(contentInformation.getHash()); @@ -194,7 +207,7 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema metadataFormat.setMetadataNamespace(metadataNamespace); metadataFormatDao.save(metadataFormat); } catch (IOException ex) { - String message = "Failed to read metadata document from input stream."; + String message = LOG_ERROR_READ_METADATA_DOCUMENT; LOG.error(message, ex); throw new UnprocessableEntityException(message); } @@ -203,13 +216,24 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema return migrateToMetadataSchemaRecord(applicationProperties, createResource, true); } + /** + * Update schema document. + * + * @param applicationProperties Settings of repository. + * @param resourceId ID of the schema document. + * @param eTag E-Tag of the current schema document. + * @param recordDocument Record of the schema. + * @param schemaDocument Schema document. + * @param supplier Method for creating access URL. + * @return Record of updated schema document. + */ public static MetadataSchemaRecord updateMetadataSchemaRecord(MetastoreConfiguration applicationProperties, String resourceId, String eTag, MultipartFile recordDocument, MultipartFile schemaDocument, - Function supplier) { - MetadataSchemaRecord record = null; + UnaryOperator supplier) { + MetadataSchemaRecord metadataRecord = null; // Do some checks first. if ((recordDocument == null || recordDocument.isEmpty()) && (schemaDocument == null || schemaDocument.isEmpty())) { @@ -219,7 +243,7 @@ public static MetadataSchemaRecord updateMetadataSchemaRecord(MetastoreConfigura } if (!(recordDocument == null || recordDocument.isEmpty())) { try { - record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchemaRecord.class); + metadataRecord = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchemaRecord.class); } catch (IOException ex) { String message = "Can't map record document to MetadataSchemaRecord"; if (ex instanceof JsonParseException) { @@ -235,10 +259,10 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema LOG.trace("Checking provided ETag."); ControllerUtils.checkEtag(eTag, dataResource); SchemaRecord schemaRecord = schemaRecordDao.findFirstBySchemaIdOrderByVersionDesc(dataResource.getId()); - if (record != null) { - record.setSchemaVersion(schemaRecord.getVersion()); + if (metadataRecord != null) { + metadataRecord.setSchemaVersion(schemaRecord.getVersion()); MetadataSchemaRecord existingRecord = migrateToMetadataSchemaRecord(applicationProperties, dataResource, false); - existingRecord = mergeRecords(existingRecord, record); + existingRecord = mergeRecords(existingRecord, metadataRecord); mergeSchemaRecord(schemaRecord, existingRecord); dataResource = migrateToDataResource(applicationProperties, existingRecord); } else { @@ -251,43 +275,49 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema ContentInformation info; info = getContentInformationOfResource(applicationProperties, dataResource); - // Check for changes... - boolean noChanges = true; - try { - byte[] currentFileContent; - File file = new File(URI.create(info.getContentUri())); - if (schemaDocument.getSize() == Files.size(file.toPath())) { - currentFileContent = FileUtils.readFileToByteArray(file); - byte[] newFileContent = schemaDocument.getBytes(); - for (int index = 0; index < currentFileContent.length; index++) { - if (currentFileContent[index] != newFileContent[index]) { - noChanges = false; - break; + + boolean noChanges = false; + String fileName = schemaDocument.getOriginalFilename(); + if (info != null) { + noChanges = true; + fileName = info.getRelativePath(); + // Check for changes... + try { + byte[] currentFileContent; + File file = new File(URI.create(info.getContentUri())); + if (schemaDocument.getSize() == Files.size(file.toPath())) { + currentFileContent = FileUtils.readFileToByteArray(file); + byte[] newFileContent = schemaDocument.getBytes(); + for (int index = 0; index < currentFileContent.length; index++) { + if (currentFileContent[index] != newFileContent[index]) { + noChanges = false; + break; + } } + } else { + noChanges = false; } - } else { - noChanges = false; + } catch (IOException ex) { + LOG.error("Error reading current file!", ex); + throw new BadArgumentException("Error reading schema document!"); } - } catch (IOException ex) { - LOG.error("Error reading current file!", ex); - throw new BadArgumentException("Error reading schema document!"); } - if (noChanges == false) { + if (!noChanges) { // Everything seems to be fine update document and increment version LOG.trace("Updating schema document (and increment version)..."); String version = dataResource.getVersion(); if (version != null) { dataResource.setVersion(Long.toString(Long.parseLong(version) + 1l)); } - ContentInformation addFile = ContentDataUtils.addFile(applicationProperties, dataResource, schemaDocument, info.getRelativePath(), null, true, supplier); + ContentDataUtils.addFile(applicationProperties, dataResource, schemaDocument, fileName, null, true, supplier); } else { schemaRecordDao.delete(schemaRecord); } } else { schemaRecordDao.delete(schemaRecord); // validate if document is still valid due to changed record settings. - record = migrateToMetadataSchemaRecord(applicationProperties, dataResource, false); - URI schemaDocumentUri = URI.create(record.getSchemaDocumentUri()); + metadataRecord = migrateToMetadataSchemaRecord(applicationProperties, dataResource, false); + URI schemaDocumentUri = URI.create(metadataRecord.getSchemaDocumentUri()); Path schemaDocumentPath = Paths.get(schemaDocumentUri); if (!Files.exists(schemaDocumentPath) || !Files.isRegularFile(schemaDocumentPath) || !Files.isReadable(schemaDocumentPath)) { @@ -308,12 +338,28 @@ record = migrateToMetadataSchemaRecord(applicationProperties, dataResource, fals return migrateToMetadataSchemaRecord(applicationProperties, dataResource, true); } + /** + * Delete schema document. + * + * @param applicationProperties Settings of repository. + * @param id ID of the schema document. + * @param eTag E-Tag of the current schema document. + * @param supplier Method for creating access URL. + */ public static void deleteMetadataSchemaRecord(MetastoreConfiguration applicationProperties, String id, String eTag, - Function supplier) { - DataResourceUtils.deleteResource(applicationProperties, id, eTag, supplier); + UnaryOperator supplier) { List listOfSchemaIds = schemaRecordDao.findBySchemaIdOrderByVersionDesc(id); + // Test for linked metadata documents + for (SchemaRecord item : listOfSchemaIds) { + List findBySchemaId = dataRecordDao.findBySchemaId(item.getSchemaId()); + if (!findBySchemaId.isEmpty()) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "Conflict with existing metadata documents."); + } + } + DataResourceUtils.deleteResource(applicationProperties, id, eTag, supplier); + listOfSchemaIds = schemaRecordDao.findBySchemaIdOrderByVersionDesc(id); for (SchemaRecord item : listOfSchemaIds) { LOG.trace("Delete entry for path '{}'", item.getSchemaDocumentUri()); List findByPath = url2PathDao.findByPath(item.getSchemaDocumentUri()); @@ -324,6 +370,13 @@ public static void deleteMetadataSchemaRecord(MetastoreConfiguration application schemaRecordDao.deleteAll(listOfSchemaIds); } + /** + * Migrate from metadata schema record to data resource. + * + * @param applicationProperties Configuration properties. + * @param metadataSchemaRecord Schema record of digital object. + * @return Data resource of digital object. + */ public static DataResource migrateToDataResource(RepoBaseConfiguration applicationProperties, MetadataSchemaRecord metadataSchemaRecord) { DataResource dataResource = null; @@ -371,7 +424,6 @@ public static DataResource migrateToDataResource(RepoBaseConfiguration applicati String defaultTitle = metadataSchemaRecord.getMimeType(); boolean titleExists = false; for (Title title : dataResource.getTitles()) { -// if (title.getTitleType() == Title.TYPE.OTHER && title.getValue().equals(defaultTitle)) { if (title.getTitleType() == Title.TYPE.OTHER) { title.setValue(defaultTitle); titleExists = true; @@ -480,7 +532,7 @@ public static MetadataSchemaRecord migrateToMetadataSchemaRecord(RepoBaseConfigu DataResource dataResource, boolean provideETag) { MetadataSchemaRecord metadataSchemaRecord = new MetadataSchemaRecord(); - long nano = 0, nano1 = 0, nano2 = 0, nano3 = 0, nano4 = 0, nano5 = 0, nano6 = 0; + long nano1 = 0, nano2 = 0, nano3 = 0, nano4 = 0, nano5 = 0, nano6 = 0; if (dataResource != null) { nano1 = System.nanoTime() / 1000000; if (provideETag) { @@ -488,8 +540,14 @@ public static MetadataSchemaRecord migrateToMetadataSchemaRecord(RepoBaseConfigu } metadataSchemaRecord.setSchemaId(dataResource.getId()); nano2 = System.nanoTime() / 1000000; - MetadataSchemaRecord.SCHEMA_TYPE schemaType = MetadataSchemaRecord.SCHEMA_TYPE.valueOf(dataResource.getFormats().iterator().next()); - metadataSchemaRecord.setType(schemaType); + try { + MetadataSchemaRecord.SCHEMA_TYPE schemaType = MetadataSchemaRecord.SCHEMA_TYPE.valueOf(dataResource.getFormats().iterator().next()); + metadataSchemaRecord.setType(schemaType); + } catch (Exception ex) { + String message = "Not a schema resource id. Returning HTTP BAD_REQUEST."; + LOG.error(message); + throw new BadArgumentException(message); + } nano3 = System.nanoTime() / 1000000; metadataSchemaRecord.setMimeType(dataResource.getTitles().iterator().next().getValue()); nano4 = System.nanoTime() / 1000000; @@ -523,11 +581,11 @@ public static MetadataSchemaRecord migrateToMetadataSchemaRecord(RepoBaseConfigu Long schemaVersion = 1l; if (dataResource.getVersion() != null) { - schemaVersion = Long.parseLong(dataResource.getVersion()); + schemaVersion = Long.valueOf(dataResource.getVersion()); } metadataSchemaRecord.setSchemaVersion(schemaVersion); - SchemaRecord schemaRecord = null; + SchemaRecord schemaRecord; try { LOG.debug("findByIDAndVersion {},{}", dataResource.getId(), metadataSchemaRecord.getSchemaVersion()); schemaRecord = schemaRecordDao.findBySchemaIdAndVersion(dataResource.getId(), metadataSchemaRecord.getSchemaVersion()); @@ -565,7 +623,9 @@ public static MetadataSchemaRecord migrateToMetadataSchemaRecord(RepoBaseConfigu LOG.trace("Unknown description type: '{}' -> skipped", nextDescription.getType()); } } - LOG.info("Migrate to schema record, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", nano1, nano2 - nano1, nano3 - nano1, nano4 - nano1, nano4 - nano1, nano6 - nano1, nano6 - nano1, nano7 - nano1, provideETag); + if (LOG.isTraceEnabled()) { + LOG.trace("Migrate to schema record, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", nano1, nano2 - nano1, nano3 - nano1, nano4 - nano1, nano4 - nano1, nano5 - nano1, nano6 - nano1, nano7 - nano1, provideETag); + } return metadataSchemaRecord; } @@ -599,9 +659,9 @@ private static ContentInformation getContentInformationOfResource(RepoBaseConfig * Validate metadata document with given schema. In case of an error a runtime * exception is thrown. * - * @param metastoreProperties - * @param document document to validate. - * @param identifier identifier of schema. + * @param metastoreProperties Configuration properties. + * @param document Document to validate. + * @param identifier Identifier of schema. * @param version Version of the document. */ public static void validateMetadataDocument(MetastoreConfiguration metastoreProperties, @@ -611,8 +671,6 @@ public static void validateMetadataDocument(MetastoreConfiguration metastoreProp SchemaRecord schemaRecord = getSchemaRecord(identifier, version); try { validateMetadataDocument(metastoreProperties, document, schemaRecord); - } catch (Throwable tw) { - throw tw; } finally { cleanUp(schemaRecord); } @@ -697,6 +755,11 @@ public static SchemaRecord getSchemaRecord(ResourceIdentifier identifier, Long v return schemaRecord; } + /** + * Remove all downloaded files for schema Record. + * + * @param schemaRecord Schema record. + */ public static void cleanUp(SchemaRecord schemaRecord) { LOG.trace("Clean up {}", schemaRecord); if (schemaRecord == null || schemaRecord.getSchemaDocumentUri() == null) { @@ -707,11 +770,12 @@ public static void cleanUp(SchemaRecord schemaRecord) { List findByUrl = url2PathDao.findByPath(pathToSchemaDocument); if (findByUrl.isEmpty()) { if (LOG.isTraceEnabled()) { - LOG.trace("-----------------------------------------"); - url2PathDao.findAll().forEach((item) -> { - LOG.trace("- {}", item); - }); - LOG.trace("-----------------------------------------"); + LOG.trace(LOG_SEPARATOR); + Page page = url2PathDao.findAll(PageRequest.of(0, 100)); + LOG.trace("List '{}' of '{}'", page.getSize(), page.getTotalElements()); + LOG.trace(LOG_SEPARATOR); + page.getContent().forEach(item -> LOG.trace("- {}", item)); + LOG.trace(LOG_SEPARATOR); } // Remove downloaded file String uri = schemaRecord.getSchemaDocumentUri(); @@ -725,9 +789,9 @@ public static void cleanUp(SchemaRecord schemaRecord) { * Validate metadata document with given schema. In case of an error a runtime * exception is thrown. * - * @param metastoreProperties - * @param document document to validate. - * @param schemaId schemaId of schema. + * @param metastoreProperties Configuration properties. + * @param document Document to validate. + * @param schemaId SchemaId of schema. * @param version Version of the document. */ public static void validateMetadataDocument(MetastoreConfiguration metastoreProperties, @@ -754,27 +818,26 @@ public static void validateMetadataDocument(MetastoreConfiguration metastoreProp * Validate metadata document with given schema. In case of an error a runtime * exception is thrown. * - * @param metastoreProperties - * @param document document to validate. - * @param schemaRecord record of the schema. + * @param metastoreProperties Configuration properties. + * @param document Document to validate. + * @param schemaRecord Record of the schema. */ public static void validateMetadataDocument(MetastoreConfiguration metastoreProperties, MultipartFile document, SchemaRecord schemaRecord) { LOG.trace("validateMetadataDocument {},{}, {}", metastoreProperties, schemaRecord, document); - long nano1 = System.nanoTime() / 1000000; if (document == null || document.isEmpty()) { String message = "Missing metadata document in body. Returning HTTP BAD_REQUEST."; LOG.error(message); throw new BadArgumentException(message); } try { - try ( InputStream inputStream = document.getInputStream()) { + try (InputStream inputStream = document.getInputStream()) { validateMetadataDocument(metastoreProperties, inputStream, schemaRecord); } } catch (IOException ex) { - String message = "Failed to read metadata document from input stream."; + String message = LOG_ERROR_READ_METADATA_DOCUMENT; LOG.error(message, ex); throw new UnprocessableEntityException(message); } @@ -784,9 +847,9 @@ public static void validateMetadataDocument(MetastoreConfiguration metastoreProp * Validate metadata document with given schema. In case of an error a runtime * exception is thrown. * - * @param metastoreProperties - * @param inputStream document to validate. - * @param schemaRecord record of the schema. + * @param metastoreProperties Configuration properties. + * @param inputStream Document to validate. + * @param schemaRecord Record of the schema. */ public static void validateMetadataDocument(MetastoreConfiguration metastoreProperties, InputStream inputStream, @@ -831,7 +894,7 @@ public static void validateMetadataDocument(MetastoreConfiguration metastoreProp throw new UnprocessableEntityException(applicableValidator.getErrorMessage()); } long nano5 = System.nanoTime() / 1000000; - LOG.info("Validate document(schemaRecord), {}, {}, {}, {}, {}, {}", nano1, nano2 - nano1, nano3 - nano1, nano4 - nano1, nano4 - nano1); + LOG.info("Validate document(schemaRecord), {}, {}, {}, {}, {}, {}", nano1, nano2 - nano1, nano3 - nano1, nano4 - nano1, nano5 - nano1); } LOG.trace("Metadata document validation succeeded."); } @@ -851,8 +914,13 @@ public static MetadataSchemaRecord getRecordByIdAndVersion(MetastoreConfiguratio //if security enabled, check permission -> if not matching, return HTTP UNAUTHORIZED or FORBIDDEN long nano = System.nanoTime() / 1000000; MetadataSchemaRecord result = null; - Page dataResource = metastoreProperties.getDataResourceService().findAllVersions(recordId, null); -// DataResource dataResource = metastoreProperties.getDataResourceService().findByAnyIdentifier(recordId, version); + Page dataResource; + try { + dataResource = metastoreProperties.getDataResourceService().findAllVersions(recordId, null); + } catch (ResourceNotFoundException rnfe) { + rnfe.setDetail("Schema document with ID '" + recordId + "' doesn't exist!"); + throw rnfe; + } long nano2 = System.nanoTime() / 1000000; Stream stream = dataResource.get(); if (version != null) { @@ -862,112 +930,50 @@ public static MetadataSchemaRecord getRecordByIdAndVersion(MetastoreConfiguratio if (findFirst.isPresent()) { result = migrateToMetadataSchemaRecord(metastoreProperties, findFirst.get(), supportEtag); } else { - String message = String.format("ID '%s' or version '%d' doesn't exist!", recordId, version.longValue()); + String message = String.format("Version '%d' of ID '%s' doesn't exist!",version, recordId); LOG.error(message); - throw new BadArgumentException(message); + throw new ResourceNotFoundException(message); } long nano3 = System.nanoTime() / 1000000; LOG.info("getRecordByIdAndVersion {}, {}, {}", nano, (nano2 - nano), (nano3 - nano)); return result; } + /** + * Merge setting from 'provided' to 'managed'. + * + * @param managed Record containing new settings. + * @param provided Record containing former settings. + * @return Record with new settings. + */ public static MetadataSchemaRecord mergeRecords(MetadataSchemaRecord managed, MetadataSchemaRecord provided) { - if (provided != null) { - // update pid - if (!Objects.isNull(provided.getPid())) { - if (!provided.getPid().equals(managed.getPid())) { - LOG.trace("Updating pid from {} to {}.", managed.getPid(), provided.getPid()); - managed.setPid(provided.getPid()); - } - } + if (provided != null && managed != null) { + //update pid + managed.setPid(mergeEntry("Update record->pid", managed.getPid(), provided.getPid())); //update acl - if (!provided.getAcl().isEmpty()) { - if (!provided.getAcl().equals(managed.getAcl())) { - // check for special access rights - // - only administrators are allowed to change ACL - if (MetadataRecordUtil.checkAccessRights(managed.getAcl())) { - // - at least principal has to remain as ADMIN - if (MetadataRecordUtil.checkAccessRights(provided.getAcl())) { - LOG.trace("Updating record acl from {} to {}.", managed.getAcl(), provided.getAcl()); - managed.setAcl(provided.getAcl()); - } - } - } - } - //update mimetype - if (provided.getMimeType() != null) { - if (!provided.getMimeType().equals(managed.getMimeType())) { - LOG.trace("Updating record mimetype from {} to {}.", managed.getMimeType(), provided.getMimeType()); - managed.setMimeType(provided.getMimeType()); - } - } - //update type - if (provided.getType() != null) { - if (!provided.getType().equals(managed.getType())) { - LOG.trace("Updating record type from {} to {}.", managed.getType(), provided.getType()); - managed.setType(provided.getType()); - } - } - //update label - if (provided.getLabel() != null) { - if (!provided.getLabel().equals(managed.getLabel())) { - LOG.trace("Updating record label from {} to {}.", managed.getLabel(), provided.getLabel()); - managed.setLabel(checkForEmptyString(provided.getLabel())); - } - } - //update definition - if (provided.getDefinition() != null) { - if (!provided.getDefinition().equals(managed.getDefinition())) { - LOG.trace("Updating record definition from {} to {}.", managed.getDefinition(), provided.getDefinition()); - managed.setDefinition(checkForEmptyString(provided.getDefinition())); - } - } - //update comment - if (provided.getComment() != null) { - if (!provided.getComment().equals(managed.getComment())) { - LOG.trace("Updating record comment from {} to {}.", managed.getComment(), provided.getComment()); - managed.setComment(checkForEmptyString(provided.getComment())); - } - } - //update doNotSync - if (!provided.getDoNotSync().equals(managed.getDoNotSync())) { - LOG.trace("Updating record comment from {} to {}.", managed.getDoNotSync(), provided.getDoNotSync()); - managed.setDoNotSync(provided.getDoNotSync()); - } + managed.setAcl(mergeAcl(managed.getAcl(), provided.getAcl())); + // update mimetype + managed.setMimeType(mergeEntry("Updating record->mimetype", managed.getMimeType(), provided.getMimeType())); + // update type + managed.setType(mergeEntry("Updating record->type", managed.getType(), provided.getType())); + // update label + managed.setLabel(mergeEntry("Updating record->label", managed.getLabel(), provided.getLabel(), true)); + // update definition + managed.setDefinition(mergeEntry("Updating record->definition", managed.getDefinition(), provided.getDefinition(), true)); + // update comment + managed.setComment(mergeEntry("Updating record->comment", managed.getComment(), provided.getComment(), true)); + // update doNotSync + managed.setDoNotSync(mergeEntry("Updating record->doNotSync", managed.getDoNotSync(), provided.getDoNotSync())); //update schemaId - if (provided.getSchemaId() != null) { - if (!provided.getSchemaId().equals(managed.getSchemaId())) { - LOG.trace("Updating record comment from {} to {}.", managed.getSchemaId(), provided.getSchemaId()); - managed.setSchemaId(provided.getSchemaId()); - } - } + managed.setSchemaId(mergeEntry("Updating record->schema", managed.getSchemaId(), provided.getSchemaId())); //update schemaVersion - if (provided.getSchemaVersion() != null) { - if (!provided.getSchemaVersion().equals(managed.getSchemaVersion())) { - LOG.trace("Updating record comment from {} to {}.", managed.getSchemaVersion(), provided.getSchemaVersion()); - managed.setSchemaVersion(provided.getSchemaVersion()); - } - } + managed.setSchemaVersion(mergeEntry("Updating record->schemaVersion", managed.getSchemaVersion(), provided.getSchemaVersion())); + } else { + managed = (managed != null) ? managed : provided; } -// LOG.trace("Setting lastUpdate to now()."); -// managed.setLastUpdate(Instant.now()); return managed; } - /** - * Check for empty String. If String is empty return 'NULL'. - * - * @param string String to check. - * @return String or 'NULL' - */ - private static String checkForEmptyString(String string) { - String returnValue = null; - if (!string.isEmpty()) { - returnValue = string; - } - return returnValue; - } - private static void validateMetadataSchemaDocument(MetastoreConfiguration metastoreProperties, SchemaRecord schemaRecord, MultipartFile document) { LOG.debug("Validate metadata schema document..."); if (document == null || document.isEmpty()) { @@ -978,7 +984,7 @@ private static void validateMetadataSchemaDocument(MetastoreConfiguration metast try { validateMetadataSchemaDocument(metastoreProperties, schemaRecord, document.getBytes()); } catch (IOException ex) { - String message = "Failed to read metadata document from input stream."; + String message = LOG_ERROR_READ_METADATA_DOCUMENT; LOG.error(message, ex); throw new UnprocessableEntityException(message); } @@ -1003,19 +1009,19 @@ private static void validateMetadataSchemaDocument(MetastoreConfiguration metast } else { LOG.trace("Validator found. Checking provided schema file."); LOG.trace("Performing validation of metadata document using schema {}, version {} and validator {}.", schemaRecord.getSchemaId(), schemaRecord.getVersion(), applicableValidator); - try ( InputStream inputStream = new ByteArrayInputStream(document)) { + try (InputStream inputStream = new ByteArrayInputStream(document)) { if (!applicableValidator.isSchemaValid(inputStream)) { String message = "Metadata schema document validation failed. Returning HTTP UNPROCESSABLE_ENTITY."; LOG.warn(message); if (LOG.isTraceEnabled()) { - LOG.trace("Schema: " + document); + LOG.trace("Schema: \n'{}'", new String(document, StandardCharsets.UTF_8)); } throw new UnprocessableEntityException(message); } } } } catch (IOException ex) { - String message = "Failed to read metadata document from input stream."; + String message = LOG_ERROR_READ_METADATA_DOCUMENT; LOG.error(message, ex); throw new UnprocessableEntityException(message); } @@ -1047,6 +1053,8 @@ private static IValidator getValidatorForRecord(MetastoreConfiguration metastore } /** + * Set the DAO for SchemaRecord. + * * @param aSchemaRecordDao the schemaRecordDao to set */ public static void setSchemaRecordDao(ISchemaRecordDao aSchemaRecordDao) { @@ -1054,14 +1062,16 @@ public static void setSchemaRecordDao(ISchemaRecordDao aSchemaRecordDao) { } /** - * @param aSchemaRecordDao the schemaRecordDao to set + * Set the DAO for MetadataFormat. + * + * @param aMetadataFormatDao the metadataFormatDao to set */ public static void setMetadataFormatDao(IMetadataFormatDao aMetadataFormatDao) { metadataFormatDao = aMetadataFormatDao; } private static void saveNewSchemaRecord(MetadataSchemaRecord result) { - SchemaRecord schemaRecord = null; + SchemaRecord schemaRecord; // Create shortcut for access. LOG.trace("Save new schema record!"); @@ -1108,22 +1118,28 @@ public static ResourceIdentifier getSchemaIdentifier(MetastoreConfiguration appl ResourceIdentifier returnValue = metadataRecord.getSchema(); if (metadataRecord.getSchema().getIdentifierType() == IdentifierType.INTERNAL) { MetadataSchemaRecord msr = MetadataSchemaRecordUtil.getRecordByIdAndVersion(applicationProperties, metadataRecord.getSchema().getIdentifier(), metadataRecord.getSchemaVersion()); - returnValue = getSchemaIdentifier(applicationProperties, msr); + returnValue = getSchemaIdentifier(msr); } return returnValue; } - public static ResourceIdentifier getSchemaIdentifier(MetastoreConfiguration applicationProperties, - MetadataSchemaRecord metadataSchemaRecord) { + /** + * Get schema identifier of schema record. + * + * @param metadataSchemaRecord Schema record of schema document. + * @return Schema identifier. + */ + public static ResourceIdentifier getSchemaIdentifier(MetadataSchemaRecord metadataSchemaRecord) { LOG.trace("Get schema identifier for '{}'.", metadataSchemaRecord); ResourceIdentifier returnValue; if (LOG.isTraceEnabled()) { LOG.trace("Looking for path '{}'", metadataSchemaRecord.getSchemaDocumentUri()); - LOG.trace("-----------------------------------------"); - url2PathDao.findAll().forEach((item) -> { - LOG.trace("- {}", item); - }); - LOG.trace("-----------------------------------------"); + LOG.trace(LOG_SEPARATOR); + Page page = url2PathDao.findAll(PageRequest.of(0, 100)); + LOG.trace("List '{}' of '{}'", page.getSize(), page.getTotalElements()); + LOG.trace(LOG_SEPARATOR); + page.getContent().forEach(item -> LOG.trace("- {}", item)); + LOG.trace(LOG_SEPARATOR); } List findByPath = url2PathDao.findByPath(metadataSchemaRecord.getSchemaDocumentUri()); if (findByPath.isEmpty()) { @@ -1135,11 +1151,16 @@ public static ResourceIdentifier getSchemaIdentifier(MetastoreConfiguration appl return returnValue; } - public static void updateMetadataFormat(MetadataSchemaRecord record) { - Optional metadataFormat = metadataFormatDao.findById(record.getSchemaId()); + /** + * Update database entry holding prefix, namespace and URL. + * + * @param metadataRecord Record holding information about schema document. + */ + public static void updateMetadataFormat(MetadataSchemaRecord metadataRecord) { + Optional metadataFormat = metadataFormatDao.findById(metadataRecord.getSchemaId()); if (metadataFormat.isPresent()) { MetadataFormat mf = metadataFormat.get(); - mf.setSchema(record.getSchemaDocumentUri()); + mf.setSchema(metadataRecord.getSchemaDocumentUri()); metadataFormatDao.save(mf); } @@ -1151,7 +1172,6 @@ public static void updateMetadataFormat(MetadataSchemaRecord record) { * @param metastoreProperties Configuration for accessing services * @param schema Identifier of the schema. * @return MetadataSchemaRecord ResponseEntity in case of an error. - * @throws IOException Error reading document. */ public static MetadataSchemaRecord getCurrentSchemaRecord(MetastoreConfiguration metastoreProperties, ResourceIdentifier schema) { @@ -1182,12 +1202,23 @@ public static long getNoOfSchemas() { } /** + * Set the DAO holding url and paths. + * * @param aUrl2PathDao the url2PathDao to set */ public static void setUrl2PathDao(IUrl2PathDao aUrl2PathDao) { url2PathDao = aUrl2PathDao; } + /** + * Set DAO for data record. + * + * @param aDataRecordDao the dataRecordDao to set + */ + public static void setDataRecordDao(IDataRecordDao aDataRecordDao) { + dataRecordDao = aDataRecordDao; + } + /** * Fix relative URI. * @@ -1210,4 +1241,44 @@ public static String fixRelativeURI(String uri) { return returnValue; } + /** + * Fix local document URI to URL. + * + * @param schemaRecord record holding schemaId and version of local document. + */ + public static final void fixSchemaDocumentUri(MetadataSchemaRecord schemaRecord) { + fixSchemaDocumentUri(schemaRecord, false); + } + + /** + * Fix local document URI to URL. + * + * @param schemaRecord record holding schemaId and version of local document. + * @param saveUrl save path to file for URL. + */ + public static final void fixSchemaDocumentUri(MetadataSchemaRecord schemaRecord, boolean saveUrl) { + String schemaDocumentUri = schemaRecord.getSchemaDocumentUri(); + schemaRecord.setSchemaDocumentUri(getSchemaDocumentUri(schemaRecord).toString()); + LOG.trace("Fix schema document Uri '{}' -> '{}'", schemaDocumentUri, schemaRecord.getSchemaDocumentUri()); + if (saveUrl) { + LOG.trace("Store path for URI!"); + Url2Path url2Path = new Url2Path(); + url2Path.setPath(schemaDocumentUri); + url2Path.setUrl(schemaRecord.getSchemaDocumentUri()); + url2Path.setType(schemaRecord.getType()); + url2Path.setVersion(schemaRecord.getSchemaVersion()); + url2PathDao.save(url2Path); + } + } + + /** + * Get URI for accessing schema document via schemaId and version. + * + * @param schemaRecord Record holding schemaId and version. + * @return URI for accessing schema document. + */ + public static final URI getSchemaDocumentUri(MetadataSchemaRecord schemaRecord) { + return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(SchemaRegistryControllerImpl.class).getSchemaDocumentById(schemaRecord.getSchemaId(), schemaRecord.getSchemaVersion(), null, null)).toUri(); + } + } diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/SchemaUtils.java b/src/main/java/edu/kit/datamanager/metastore2/util/SchemaUtils.java index 4b044d17..6596d949 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/SchemaUtils.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/SchemaUtils.java @@ -18,6 +18,7 @@ import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -32,6 +33,7 @@ import org.xml.sax.SAXException; /** + * Utility class for (XML) schema documents. * * @author jejkal */ @@ -41,9 +43,13 @@ public class SchemaUtils { private static final int MAX_LENGTH_OF_HEADER = 100; - private static final Pattern JSON_FIRST_BYTE = Pattern.compile("(\\R\\s)*\\s*\\{\\s*\"\\$(.|\\s)*");//^\\s{\\s*\".*"); + private static final Pattern JSON_FIRST_BYTE = Pattern.compile("(\\R\\s)*\\s*\\{\\s*\"\\$(.|\\s)*");// private static final Pattern XML_FIRST_BYTE = Pattern.compile("((.|\\s)*<\\?xml[^<]*)?\\s*<\\s*(\\w{2,3}:)?schema(.|\\s)*", Pattern.MULTILINE); + SchemaUtils() { + //Utility class + } + /** * Guess type of schema document. * @@ -54,7 +60,7 @@ public static MetadataSchemaRecord.SCHEMA_TYPE guessType(byte[] schema) { // Cut schema to a maximum of MAX_LENGTH_OF_HEADER characters. if (schema != null) { int length = schema.length > MAX_LENGTH_OF_HEADER ? MAX_LENGTH_OF_HEADER : schema.length; - String schemaAsString = new String(schema, 0, length); + String schemaAsString = new String(schema, 0, length, StandardCharsets.UTF_8); LOG.trace("Guess type for '{}'", schemaAsString); Matcher m = JSON_FIRST_BYTE.matcher(schemaAsString); @@ -74,7 +80,12 @@ public static MetadataSchemaRecord.SCHEMA_TYPE guessType(byte[] schema) { return null; } - public static String getTargetNamespaceFromSchema(byte[] schema) { + /** + * Determine target namespace from schema. + * @param schema Schema document. + * @return Namespace. + */ + public static String getTargetNamespaceFromSchema(byte[] schema) { String namespace = null; try { DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); @@ -83,7 +94,7 @@ public static String getTargetNamespaceFromSchema(byte[] schema) { documentBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); Document document = documentBuilder.parse(new ByteArrayInputStream(schema)); - NamedNodeMap map = ((Element) document.getDocumentElement()).getAttributes(); + NamedNodeMap map = document.getDocumentElement().getAttributes(); namespace = map.getNamedItem("targetNamespace").getNodeValue(); } catch (ParserConfigurationException | SAXException | IOException ex) { java.util.logging.Logger.getLogger(SchemaUtils.class.getName()).log(Level.SEVERE, null, ex); diff --git a/src/main/java/edu/kit/datamanager/metastore2/validation/IValidator.java b/src/main/java/edu/kit/datamanager/metastore2/validation/IValidator.java index 0ddfc6e7..9295868a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/validation/IValidator.java +++ b/src/main/java/edu/kit/datamanager/metastore2/validation/IValidator.java @@ -1,7 +1,17 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Copyright 2018 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package edu.kit.datamanager.metastore2.validation; @@ -17,18 +27,19 @@ public interface IValidator { /** * Get an instance of the validator. - * @return + * + * @return instance of validator. */ - default IValidator getInstance(){ + default IValidator getInstance() { return this; } + /** * Supports the given schema type. * * @see MetadataSchemaRecord#type - * * @param type Type of the schema. - * + * * @return supports schema type or not. */ boolean supportsSchemaType(MetadataSchemaRecord.SCHEMA_TYPE type); @@ -48,6 +59,7 @@ default IValidator getInstance(){ * * @param schemaFile File containing schema. * @param metadataDocumentStream Stream containing metadata document. + * * @return valid or not. */ boolean validateMetadataDocument(File schemaFile, InputStream metadataDocumentStream); diff --git a/src/main/java/edu/kit/datamanager/metastore2/validation/impl/XmlValidator.java b/src/main/java/edu/kit/datamanager/metastore2/validation/impl/XmlValidator.java index 8efc572a..51607ed0 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/validation/impl/XmlValidator.java +++ b/src/main/java/edu/kit/datamanager/metastore2/validation/impl/XmlValidator.java @@ -39,101 +39,98 @@ */ @Component public class XmlValidator implements IValidator { - - private static final Logger LOG = LoggerFactory.getLogger(XmlValidator.class); - - private String errorMessage; - - @Override - public boolean supportsSchemaType(MetadataSchemaRecord.SCHEMA_TYPE type) { - return MetadataSchemaRecord.SCHEMA_TYPE.XML.equals(type); - } - - @Override - public IValidator getInstance() { - return new XmlValidator(); - } - - @Override - public boolean isSchemaValid(InputStream schemaStream) { - boolean result = false; - LOG.trace("Checking schema for validity."); - try { -// SchemaFactory schemaFactory = getSchemaFactory(); - SAXParser saxParser = getSaxParser(); - DefaultHandler errorHandler = new DefaultHandler(); - saxParser.parse(schemaStream, errorHandler); - - LOG.trace("Schema seems to be valid."); - result = true; - } catch (SAXException | IOException e) { - LOG.error("Failed to validate schema.", e); - errorMessage = new String("Validation error: " + e.getMessage()); - } - return result; - } - - @Override - public boolean validateMetadataDocument(File schemaFile, InputStream metadataDocumentStream) { - boolean valid = false; - LOG.trace("Checking metdata document using schema at {}.", schemaFile); - LOG.trace("Reading metadata document from stream."); - try { - SchemaFactory schemaFactory = getSchemaFactory(); - - LOG.trace("Creating schema instance."); - Schema schema = null; - schema = schemaFactory.newSchema(schemaFile); - - LOG.trace("Obtaining validator."); - Validator validator = schema.newValidator(); - - LOG.trace("Validating metadata file."); - Source xmlFile = new StreamSource(metadataDocumentStream); - validator.validate(xmlFile); - - LOG.trace("Metadata document is valid according to schema."); - valid = true; - } catch (SAXException | IOException e) { - LOG.error("Failed to validate metadata document.", e); - errorMessage = new String("Validation error: " + e.getMessage()); - } - return valid; - } - - @Override - public String getErrorMessage() { - return errorMessage; - } - /** - * Get schema factory with disabled DTD parsing due to XXE vulnerabilty. - * - * @return schema factory - */ - private SchemaFactory getSchemaFactory() throws SAXNotRecognizedException, SAXNotSupportedException { - SchemaFactory schemaFactory; - schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); -// schemaFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - return schemaFactory; - } - - private SAXParser getSaxParser() { - SAXParser parser = null; - try { - SAXParserFactory spf = SAXParserFactory.newInstance(); - spf.setValidating(true); - spf.setNamespaceAware(true); - spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.SCHEMA_VALIDATION_FEATURE, true); - spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.DISALLOW_DOCTYPE_DECL_FEATURE, true); - spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.LOAD_EXTERNAL_DTD_FEATURE, false); - spf.setFeature(Constants.SAX_FEATURE_PREFIX + Constants.EXTERNAL_GENERAL_ENTITIES_FEATURE, false); - spf.setFeature(Constants.SAX_FEATURE_PREFIX + Constants.EXTERNAL_PARAMETER_ENTITIES_FEATURE, false); - spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - parser = spf.newSAXParser(); - } catch (ParserConfigurationException | SAXException ex) { - LOG.error("Error creating SAX parser!", ex); - } - return parser; + private static final Logger LOG = LoggerFactory.getLogger(XmlValidator.class); + + private String errorMessage; + + @Override + public boolean supportsSchemaType(MetadataSchemaRecord.SCHEMA_TYPE type) { + return MetadataSchemaRecord.SCHEMA_TYPE.XML.equals(type); + } + + @Override + public IValidator getInstance() { + return new XmlValidator(); + } + + @Override + public boolean isSchemaValid(InputStream schemaStream) { + boolean result = false; + LOG.trace("Checking schema for validity."); + try { + SAXParser saxParser = getSaxParser(); + DefaultHandler errorHandler = new DefaultHandler(); + saxParser.parse(schemaStream, errorHandler); + + LOG.trace("Schema seems to be valid."); + result = true; + } catch (ParserConfigurationException | SAXException | IOException e) { + LOG.error("Failed to validate schema.", e); + errorMessage = "Validation error: " + e.getMessage(); } + return result; + } + + @Override + public boolean validateMetadataDocument(File schemaFile, InputStream metadataDocumentStream) { + boolean valid = false; + LOG.trace("Checking metdata document using schema at {}.", schemaFile); + LOG.trace("Reading metadata document from stream."); + try { + SchemaFactory schemaFactory = getSchemaFactory(); + + LOG.trace("Creating schema instance."); + Schema schema; + schema = schemaFactory.newSchema(schemaFile); + + LOG.trace("Obtaining validator."); + Validator validator = schema.newValidator(); + + LOG.trace("Validating metadata file."); + Source xmlFile = new StreamSource(metadataDocumentStream); + validator.validate(xmlFile); + + LOG.trace("Metadata document is valid according to schema."); + valid = true; + } catch (SAXException | IOException e) { + LOG.error("Failed to validate metadata document.", e); + errorMessage = "Validation error: " + e.getMessage(); } + return valid; + } + + @Override + public String getErrorMessage() { + return errorMessage; + } + + /** + * Get schema factory with disabled DTD parsing due to XXE vulnerabilty. + * + * @return schema factory + */ + private SchemaFactory getSchemaFactory() { + SchemaFactory schemaFactory; + schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + + return schemaFactory; + } + + private SAXParser getSaxParser() throws ParserConfigurationException, SAXException { + SAXParser parser; + + SAXParserFactory spf = SAXParserFactory.newInstance(); + spf.setValidating(true); + spf.setNamespaceAware(true); + spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.SCHEMA_VALIDATION_FEATURE, true); + spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.DISALLOW_DOCTYPE_DECL_FEATURE, true); + spf.setFeature(Constants.XERCES_FEATURE_PREFIX + Constants.LOAD_EXTERNAL_DTD_FEATURE, false); + spf.setFeature(Constants.SAX_FEATURE_PREFIX + Constants.EXTERNAL_GENERAL_ENTITIES_FEATURE, false); + spf.setFeature(Constants.SAX_FEATURE_PREFIX + Constants.EXTERNAL_PARAMETER_ENTITIES_FEATURE, false); + spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + parser = spf.newSAXParser(); + + return parser; + } +} diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/IFrontendController.java b/src/main/java/edu/kit/datamanager/metastore2/web/IFrontendController.java index 8dd23b86..213510ae 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/IFrontendController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/IFrontendController.java @@ -8,7 +8,7 @@ import edu.kit.datamanager.metastore2.dto.TabulatorLocalPagination; import edu.kit.datamanager.metastore2.dto.TabulatorRemotePagination; import io.swagger.v3.oas.annotations.Parameter; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import org.springdoc.core.converters.models.PageableAsQueryParam; import org.springframework.data.domain.Pageable; import org.springframework.http.ResponseEntity; diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/ILandingPageController.java b/src/main/java/edu/kit/datamanager/metastore2/web/ILandingPageController.java new file mode 100644 index 00000000..ba832f14 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/metastore2/web/ILandingPageController.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.metastore2.web; + +import edu.kit.datamanager.metastore2.domain.MetadataRecord; +import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.context.request.WebRequest; + +/** + * Interface for metadata documents controller. + */ +@ApiResponses(value = { + @ApiResponse(responseCode = "401", description = "Unauthorized is returned if authorization is required but was not provided."), + @ApiResponse(responseCode = "403", description = "Forbidden is returned if the caller has no sufficient privileges.")}) +public interface ILandingPageController { + + @Operation(summary = "Get landing page of schema by schema id (and version).", description = "Show landing page by its schema id. " + + "Depending on a user's role, accessing a specific record may be allowed or forbidden. " + + "Furthermore, a specific version of the schema can be returned by providing a version number as request parameter. If no version is specified, all versions will be returned.", + responses = { + @ApiResponse(responseCode = "200", description = "OK and the landingpage is returned if the id exists and the user has sufficient permission.", content = @Content(schema = @Schema(implementation = MetadataSchemaRecord.class))), + @ApiResponse(responseCode = "404", description = "Not found is returned, if no record for the provided id and version was found.")}) + @RequestMapping(value = {"/schema-landing-page"}, method = {RequestMethod.GET}, produces = {"text/html"}) + public String getLandingPageOfSchemaWithId( + @Parameter(description = "The record identifier or schema identifier.", required = true) @RequestParam(value = "schemaId") String id, + @Parameter(description = "The version of the record.", required = false) @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr, + Model model); + + @Operation(summary = "Get a landing page by id.", description = "Obtain a single record by its resource identifier. " + + "Depending on a user's role, accessing a specific record may be allowed or forbidden. Furthermore, a specific version of the record can be returned " + + "by providing a version number as request parameter.", + responses = { + @ApiResponse(responseCode = "200", description = "OK and the record is returned if the record exists and the user has sufficient permission.", content = @Content(schema = @Schema(implementation = MetadataRecord.class))), + @ApiResponse(responseCode = "404", description = "Not found is returned, if no record for the provided id or version was found.")}) + + @RequestMapping(value = {"/metadata-landing-page"}, method = {RequestMethod.GET}, produces = {"text/html"}) + public String getLandingPageOfMetadataDocumentWithId( + @Parameter(description = "The identifier of the metadata document.", required = true) @RequestParam(value = "id") String id, + @Parameter(description = "The version of the digital object. This parameter only has an effect if versioning is enabled.", required = false) @RequestParam(value = "version") Long version, + WebRequest wr, + HttpServletResponse hsr, + Model model); +} diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataController.java b/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataController.java index a6710ec9..1c7c6788 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataController.java @@ -29,8 +29,8 @@ import java.net.URISyntaxException; import java.time.Instant; import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.springdoc.core.converters.models.PageableAsQueryParam; import org.springframework.boot.actuate.info.InfoContributor; import org.springframework.data.domain.Pageable; @@ -38,6 +38,7 @@ import org.springframework.data.web.PageableDefault; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.ui.Model; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -46,11 +47,11 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.context.request.WebRequest; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; import org.springframework.web.util.UriComponentsBuilder; /** - * - * @author jejkal + * Interface for metadata documents controller. */ @ApiResponses(value = { @ApiResponse(responseCode = "401", description = "Unauthorized is returned if authorization in required but was not provided."), @@ -66,10 +67,10 @@ public interface IMetadataController extends InfoContributor { @ApiResponse(responseCode = "404", description = "Not found is returned, if no schema for the provided schema id was found."), @ApiResponse(responseCode = "409", description = "A Conflict is returned, if there is already a record for the related resource id and the provided schema id.")}) - @RequestMapping(path = "", method = RequestMethod.POST, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}) + @RequestMapping(value = {"","/"}, method = RequestMethod.POST, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}) @ResponseBody public ResponseEntity createRecord( - @Parameter(description = "Json representation of the metadata record.", required = true) @RequestPart(name = "record", required = true) final MultipartFile record, + @Parameter(description = "Json representation of the metadata record.", required = true) @RequestPart(name = "record", required = true) final MultipartFile metadataRecord, @Parameter(description = "The metadata document associated with the record. The document must match the schema selected by the record.", required = true) @RequestPart(name = "document", required = true) final MultipartFile document, final HttpServletRequest request, final HttpServletResponse response, @@ -101,6 +102,18 @@ public ResponseEntity getAclById(@Parameter(description = "The record WebRequest wr, HttpServletResponse hsr); + @Operation(summary = "Get a landing page by id.", description = "Obtain a single record by its resource identifier. " + + "Depending on a user's role, accessing a specific record may be allowed or forbidden. Furthermore, a specific version of the record can be returned " + + "by providing a version number as request parameter.", + responses = { + @ApiResponse(responseCode = "200", description = "OK and the record is returned if the record exists and the user has sufficient permission.", content = @Content(schema = @Schema(implementation = MetadataRecord.class))), + @ApiResponse(responseCode = "404", description = "Not found is returned, if no record for the provided id or version was found.")}) + + @RequestMapping(value = {"/{id}"}, method = {RequestMethod.GET}, produces = {"text/html"}) + public ModelAndView getLandingpageById(@Parameter(description = "The record identifier or related resource identifier.", required = true) @PathVariable(value = "id") String id, + @Parameter(description = "The version of the metadata document. This parameter only has an effect if versioning is enabled.", required = false) @RequestParam(value = "version") Long version, + WebRequest wr, + HttpServletResponse hsr); @Operation(summary = "Get a metadata document by record identifier.", description = "Obtain a single metadata document identified by its resource identifier." + "Depending on a user's role, accessing a specific record may be allowed or forbidden. " + "Furthermore, a specific version of the metadata document can be returned by providing a version number as request parameter.", @@ -123,7 +136,7 @@ public ResponseEntity getMetadataDocumentById(@Parameter(description = "The reco + "If no parameters are provided, all accessible records are listed. If versioning is enabled, only the most recent version is listed (except in case of 'id' is provided).", responses = { @ApiResponse(responseCode = "200", description = "OK and a list of records or an empty list if no record matches.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = MetadataRecord.class))))}) - @RequestMapping(value = {""}, method = {RequestMethod.GET}) + @RequestMapping(value = {"", "/"}, method = {RequestMethod.GET}) @PageableAsQueryParam @ResponseBody public ResponseEntity> getRecords( @@ -150,7 +163,7 @@ public ResponseEntity> getRecords( }) ResponseEntity updateRecord( @Parameter(description = "The resource identifier.", required = true) @PathVariable("id") String id, - @Parameter(description = "JSON representation of the metadata record.", required = false) @RequestPart(name = "record", required = false) final MultipartFile record, + @Parameter(description = "JSON representation of the metadata record.", required = false) @RequestPart(name = "record", required = false) final MultipartFile metadataRecord, @Parameter(description = "The metadata document associated with the record. The document must match the schema defined in the record.", required = false) @RequestPart(name = "document", required = false) final MultipartFile document, final WebRequest request, final HttpServletResponse response, diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataEditorController.java b/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataEditorController.java index c8579a47..ce87b9cb 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataEditorController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/IMetadataEditorController.java @@ -5,7 +5,7 @@ */ package edu.kit.datamanager.metastore2.web; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponse; import org.springframework.data.domain.Pageable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.context.request.WebRequest; @@ -13,12 +13,12 @@ import org.springframework.web.util.UriComponentsBuilder; /** + * Interface for metadata editor controller. * * @author sabrinechelbi */ public interface IMetadataEditorController { - @RequestMapping("/schema-management") public ModelAndView schemaManagement(); diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/ISchemaRegistryController.java b/src/main/java/edu/kit/datamanager/metastore2/web/ISchemaRegistryController.java index bea933cd..f0c51619 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/ISchemaRegistryController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/ISchemaRegistryController.java @@ -25,10 +25,10 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.time.Instant; import java.util.List; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.springdoc.core.converters.models.PageableAsQueryParam; import org.springframework.boot.actuate.info.InfoContributor; import org.springframework.data.domain.Pageable; @@ -45,9 +45,11 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.context.request.WebRequest; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; import org.springframework.web.util.UriComponentsBuilder; /** + * Interface for schema document controller. * * @author jejkal */ @@ -62,10 +64,10 @@ public interface ISchemaRegistryController extends InfoContributor { @ApiResponse(responseCode = "201", description = "Created is returned only if the record has been validated, persisted and the document was successfully validated and stored.", content = @Content(schema = @Schema(implementation = MetadataSchemaRecord.class))), @ApiResponse(responseCode = "400", description = "Bad Request is returned if the provided metadata record is invalid or if the validation of the provided schema failed."), @ApiResponse(responseCode = "409", description = "A Conflict is returned, if there is already a record for the provided schema id.")}) - @RequestMapping(path = "", method = RequestMethod.POST, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}, produces = {MediaType.APPLICATION_JSON_VALUE}) + @RequestMapping(value = {"", "/"}, method = RequestMethod.POST, consumes = {MediaType.MULTIPART_FORM_DATA_VALUE}, produces = {MediaType.APPLICATION_JSON_VALUE}) @ResponseBody public ResponseEntity createRecord( - @Parameter(description = "Json representation of the schema record.", required = true) @RequestPart(name = "record", required = true) final MultipartFile record, + @Parameter(description = "Json representation of the schema record.", required = true) @RequestPart(name = "record", required = true) final MultipartFile schemaRecord, @Parameter(description = "The metadata schema document associated with the record.", required = true) @RequestPart(name = "schema", required = true) final MultipartFile document, final HttpServletRequest request, final HttpServletResponse response, @@ -84,6 +86,18 @@ public ResponseEntity getRecordById(@Parameter(description = "The record identif WebRequest wr, HttpServletResponse hsr); + @Operation(summary = "Get landing page of schema by schema id (and version).", description = "Show landing page by its schema id. " + + "Depending on a user's role, accessing a specific record may be allowed or forbidden. " + + "Furthermore, a specific version of the schema can be returned by providing a version number as request parameter. If no version is specified, all versions will be returned.", + responses = { + @ApiResponse(responseCode = "200", description = "OK and the landingpage is returned if the id exists and the user has sufficient permission.", content = @Content(schema = @Schema(implementation = MetadataSchemaRecord.class))), + @ApiResponse(responseCode = "404", description = "Not found is returned, if no record for the provided id and version was found.")}) + @RequestMapping(value = {"/{schemaId}"}, method = {RequestMethod.GET}, produces = {"text/html"}) + public ModelAndView getLandingPageById(@Parameter(description = "The record identifier or schema identifier.", required = true) @PathVariable(value = "schemaId") String id, + @Parameter(description = "The version of the record.", required = false) @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr); + @Operation(summary = "Validate a metadata document.", description = "Validate the provided metadata document using the addressed schema. If all parameters" + " are provided, the schema is identified uniquely by schemaId and version. If the version is omitted, the most recent version of the " + "schema is used. This endpoint returns HTTP NO_CONTENT if it succeeds. Otherwise, an error response is returned, e.g. HTTP UNPROCESSABLE_ENTITY (422) if validation fails.", @@ -122,15 +136,15 @@ public ResponseEntity getSchemaDocumentById(@Parameter(description = "The schema + "If no parameters are provided, all accessible records are listed. With regard to schema versions, only the most recent version of each schema is listed.", responses = { @ApiResponse(responseCode = "200", description = "OK and a list of records or an empty list of no record matches.", content = @Content(array = @ArraySchema(schema = @Schema(implementation = MetadataSchemaRecord.class))))}) - @RequestMapping(value = {""}, method = {RequestMethod.GET}) + @RequestMapping(value = {"", "/"}, method = {RequestMethod.GET}) @ResponseBody @PageableAsQueryParam public ResponseEntity> getRecords( - @Parameter(description = "SchemaId", required = false) @RequestParam(value = "schemaId", required = false) String schemaId, + @Parameter(description = "SchemaId", required = false) @RequestParam(value = "schemaId", required = false) String schemaId, @Parameter(description = "A list of mime types returned schemas are associated with.", required = false) @RequestParam(value = "mimeType", required = false) List mimeTypes, @Parameter(description = "The UTC time of the earliest update of a returned record.", required = false) @RequestParam(name = "from", required = false) Instant updateFrom, @Parameter(description = "The UTC time of the latest update of a returned record.", required = false) @RequestParam(name = "until", required = false) Instant updateUntil, - @Parameter(hidden = true)@PageableDefault(sort = {"id"}, direction = Sort.Direction.ASC)Pageable pgbl, + @Parameter(hidden = true) @PageableDefault(sort = {"id"}, direction = Sort.Direction.ASC) Pageable pgbl, WebRequest wr, HttpServletResponse hsr, UriComponentsBuilder ucb); @@ -150,7 +164,7 @@ public ResponseEntity> getRecords( }) ResponseEntity updateRecord( @Parameter(description = "The schema id.", required = true) @PathVariable("schemaId") final String schemaId, - @Parameter(description = "Json representation of the schema record.", required = false) @RequestPart(name = "record", required = false) final MultipartFile record, + @Parameter(description = "Json representation of the schema record.", required = false) @RequestPart(name = "record", required = false) final MultipartFile schemaRecord, @Parameter(description = "The metadata schema document associated with the record.", required = false) @RequestPart(name = "schema", required = false) final MultipartFile document, final WebRequest request, final HttpServletResponse response diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java index 1bdea6bc..15b31364 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java @@ -10,13 +10,12 @@ import edu.kit.datamanager.metastore2.dto.TabulatorLocalPagination; import edu.kit.datamanager.metastore2.dto.TabulatorRemotePagination; import edu.kit.datamanager.metastore2.web.IFrontendController; -import edu.kit.datamanager.util.AuthenticationHelper; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; import java.util.Arrays; import java.util.List; -import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -32,6 +31,7 @@ import org.springframework.web.util.UriComponentsBuilder; /** + * Controller used by web frontends. * * @author sabrinechelbi */ @@ -43,6 +43,9 @@ public class FrontendControllerImpl implements IFrontendController { private static final Logger LOG = LoggerFactory.getLogger(FrontendControllerImpl.class); + private static final String SHOW_PAGE = "Pageable: '{}'"; + private static final String CONTENT_RANGE = "Content-Range"; + @Autowired private MetadataControllerImpl metadtaControllerImpl; @@ -56,18 +59,16 @@ public ResponseEntity findAllSchemasForTabulator( final HttpServletResponse hsr, final UriComponentsBuilder ucb) { - LOG.trace("Performing findAllSchemasForTabulator()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + LOG.trace("Performing findAllSchemasForTabulator( pgbl='{}').", pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); + LOG.trace(SHOW_PAGE, pageable); ResponseEntity> responseEntity4schemaRecords = schemaControllerImpl.getRecords(null, null, null, null, pageable, wr, hsr, ucb); List schemaRecords = responseEntity4schemaRecords.getBody(); - String pageSize = responseEntity4schemaRecords.getHeaders().getFirst("Content-Range"); + String pageSize = responseEntity4schemaRecords.getHeaders().getFirst(CONTENT_RANGE); TabulatorLocalPagination tabulatorLocalPagination = TabulatorLocalPagination.builder() .lastPage(tabulatorLastPage(pageSize, pageable)) @@ -83,19 +84,17 @@ public ResponseEntity findAllMetadataForTabulator( final WebRequest wr, final HttpServletResponse hsr, final UriComponentsBuilder ucb) { - LOG.trace("Performing findAllMetadataForTabulator()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + LOG.trace("Performing findAllMetadataForTabulator( id='{}', pgbl='{}').", id, pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); - List schemaIds = id == null ? null : Arrays.asList(id); - ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, schemaIds, null, null, pageable, wr, hsr, ucb); + LOG.trace(SHOW_PAGE, pageable); + List metadataDocumentId = id == null ? null : Arrays.asList(id); + ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, metadataDocumentId, null, null, pageable, wr, hsr, ucb); List metadataRecords = responseEntity4metadataRecords.getBody(); - String pageSize = responseEntity4metadataRecords.getHeaders().getFirst("Content-Range"); + String pageSize = responseEntity4metadataRecords.getHeaders().getFirst(CONTENT_RANGE); TabulatorLocalPagination tabulatorLocalPagination = TabulatorLocalPagination.builder() .lastPage(tabulatorLastPage(pageSize, pageable)) @@ -105,21 +104,22 @@ public ResponseEntity findAllMetadataForTabulator( } @Override - public ResponseEntity getSchemaRecordsForUi(Pageable pgbl, WebRequest wr, HttpServletResponse hsr, UriComponentsBuilder ucb) { - LOG.trace("Performing getSchemaRecordsForUi()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + public ResponseEntity getSchemaRecordsForUi( + @Parameter(hidden = true) final Pageable pgbl, + final WebRequest wr, + final HttpServletResponse hsr, + final UriComponentsBuilder ucb) { + LOG.trace("Performing getSchemaRecordsForUi( pgbl='{}').", pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); + LOG.trace(SHOW_PAGE, pageable); ResponseEntity> responseEntity4schemaRecords = schemaControllerImpl.getRecords(null, null, null, null, pageable, wr, hsr, ucb); List schemaRecords = responseEntity4schemaRecords.getBody(); - String pageSize = responseEntity4schemaRecords.getHeaders().getFirst("Content-Range"); + String pageSize = responseEntity4schemaRecords.getHeaders().getFirst(CONTENT_RANGE); TabulatorRemotePagination tabulatorRemotePagination = TabulatorRemotePagination.builder() .lastPage(tabulatorLastPage(pageSize, pageable)) @@ -131,21 +131,35 @@ public ResponseEntity getSchemaRecordsForUi(Pageable } @Override - public ResponseEntity getMetadataRecordsForUi(@RequestParam(value = "id", required = false) String id, Pageable pgbl, WebRequest wr, HttpServletResponse hsr, UriComponentsBuilder ucb) { - LOG.trace("Performing getMetadataRecordsForUi()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + public ResponseEntity getMetadataRecordsForUi( + @RequestParam(value = "id", required = false) String id, + @Parameter(hidden = true) final Pageable pgbl, + final WebRequest wr, + final HttpServletResponse hsr, + final UriComponentsBuilder ucb) { + LOG.trace("Performing getMetadataRecordsForUi( id='{}', pgbl='{}').", id, pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } + LOG.trace(SHOW_PAGE, pageable); - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); List schemaIds = id == null ? null : Arrays.asList(id); - ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, schemaIds, null, null, pageable, wr, hsr, ucb); - List metadataRecords = responseEntity4metadataRecords.getBody(); - - String pageSize = responseEntity4metadataRecords.getHeaders().getFirst("Content-Range"); + ResponseEntity< List> responseEntity4metadataRecords; + List metadataRecords = null; + String pageSize = null; + try { + responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, schemaIds, null, null, pageable, wr, hsr, ucb); + metadataRecords = responseEntity4metadataRecords.getBody(); + pageSize = responseEntity4metadataRecords.getHeaders().getFirst(CONTENT_RANGE); + } catch (Exception ex) { + // Test for document id instead of schema id. + if (schemaIds != null) { + responseEntity4metadataRecords = metadtaControllerImpl.getRecords(schemaIds.get(0), null, null, null, null, pageable, wr, hsr, ucb); + metadataRecords = responseEntity4metadataRecords.getBody(); + pageSize = responseEntity4metadataRecords.getHeaders().getFirst(CONTENT_RANGE); + } + } TabulatorRemotePagination tabulatorRemotePagination = TabulatorRemotePagination.builder() .lastPage(tabulatorLastPage(pageSize, pageable)) @@ -163,6 +177,7 @@ public ResponseEntity getMetadataRecordsForUi(@Reques * @param pageable page object * @return the total number of the available pages */ + @SuppressWarnings("StringSplitter") private int tabulatorLastPage(String pageSize, Pageable pageable) { if ((Integer.parseInt(pageSize.split("/")[1]) % pageable.getPageSize()) == 0) { return Integer.parseInt(pageSize.split("/")[1]) / pageable.getPageSize(); diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/LandingPageControllerImpl.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/LandingPageControllerImpl.java new file mode 100644 index 00000000..4c9d068b --- /dev/null +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/LandingPageControllerImpl.java @@ -0,0 +1,144 @@ +/* + * Copyright 2019 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.metastore2.web.impl; + +import edu.kit.datamanager.metastore2.configuration.ApplicationProperties; +import edu.kit.datamanager.metastore2.configuration.MetastoreConfiguration; +import edu.kit.datamanager.metastore2.domain.MetadataRecord; +import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; +import edu.kit.datamanager.metastore2.domain.SchemaRecord; +import edu.kit.datamanager.metastore2.util.MetadataRecordUtil; +import edu.kit.datamanager.metastore2.util.MetadataSchemaRecordUtil; +import edu.kit.datamanager.metastore2.web.ILandingPageController; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletResponse; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.ModelAndView; + +/** + * Controller for metadata documents. + */ +@Controller +@RequestMapping(value = "") +@Tag(name = "Landing Page") +@Schema(description = "Landing page for all digital objects stored in this repo.") +public class LandingPageControllerImpl implements ILandingPageController { + + private static final Logger LOG = LoggerFactory.getLogger(LandingPageControllerImpl.class); + + private final MetastoreConfiguration metadataConfig; + + private final MetastoreConfiguration schemaConfig; + + /** + * Constructor for metadata documents controller. + * + * @param applicationProperties Configuration for controller. + * @param metadataConfig Configuration for metadata documents repository. + * @param schemaConfig Configuration for schema documents repository. + */ + public LandingPageControllerImpl(ApplicationProperties applicationProperties, + MetastoreConfiguration metadataConfig, + MetastoreConfiguration schemaConfig) { + this.metadataConfig = metadataConfig; + this.schemaConfig = schemaConfig; + LOG.info("------------------------------------------------------"); + LOG.info("------{}", this.metadataConfig); + LOG.info("------------------------------------------------------"); + } + + @Override + public String getLandingPageOfSchemaWithId(@RequestParam(value = "schemaId") String id, + @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr, + Model model) { + LOG.trace("Performing getLandingPageOfSchemaWithId({}, {}).", id, version); + + //if security is enabled, include principal in query + LOG.debug("Performing a query for records with given id."); + MetadataSchemaRecord recordByIdAndVersion = MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, id, version); + List recordList = new ArrayList<>(); + recordList.add(recordByIdAndVersion); + if (version == null) { + long totalNoOfElements = recordByIdAndVersion.getSchemaVersion(); + for (long size = totalNoOfElements - 1; size > 0; size--) { + recordList.add(MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, id, size)); + } + } + + LOG.trace("Fix URL for all schema records"); + List metadataList = new ArrayList<>(); + recordList.forEach(metadataRecord -> { + MetadataSchemaRecordUtil.fixSchemaDocumentUri(metadataRecord); + metadataList.add(metadataRecord); + }); + + model.addAttribute("records", metadataList); + + return "schema-landing-page.html"; + } + + @Override + public String getLandingPageOfMetadataDocumentWithId(@PathVariable(value = "id") String id, + @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr, + Model model + ) { + + LOG.trace("Performing getLandingPageOfMetadataDocumentWithId({}, {}).", id, version); + + //if security is enabled, include principal in query + LOG.debug("Performing a query for all records with given id..."); + MetadataRecord recordByIdAndVersion = MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, version); + List recordList = new ArrayList<>(); + + recordList.add(recordByIdAndVersion); + if (version == null) { + long totalNoOfElements = recordByIdAndVersion.getRecordVersion(); + for (long size = totalNoOfElements - 1; size > 0; size--) { + recordList.add(MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, size)); + } + } + + LOG.trace("Fix URL for all metadata records"); + List metadataList = new ArrayList<>(); + recordList.forEach(metadataRecord -> { + MetadataRecordUtil.fixMetadataDocumentUri(metadataRecord); + metadataList.add(metadataRecord); + }); + + SchemaRecord schemaRecord = MetadataSchemaRecordUtil.getSchemaRecord(metadataList.get(0).getSchema(), metadataList.get(0).getSchemaVersion()); + + model.addAttribute("type", schemaRecord.getType()); + model.addAttribute("records", metadataList); + + return "metadata-landing-page.html"; + } + +} diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataControllerImpl.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataControllerImpl.java index 47562288..e19b508d 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataControllerImpl.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataControllerImpl.java @@ -16,10 +16,6 @@ package edu.kit.datamanager.metastore2.web.impl; import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; import edu.kit.datamanager.entities.PERMISSION; import edu.kit.datamanager.entities.RepoUserRole; import edu.kit.datamanager.entities.messaging.MetadataResourceMessage; @@ -52,10 +48,10 @@ import edu.kit.datamanager.util.AuthenticationHelper; import edu.kit.datamanager.util.ControllerUtils; import io.swagger.v3.core.util.Json; -import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -67,14 +63,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import java.util.function.UnaryOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.info.Info; -import org.springframework.cloud.gateway.mvc.ProxyExchange; import org.springframework.context.annotation.Bean; import org.springframework.core.io.FileSystemResource; import org.springframework.data.domain.Page; @@ -86,19 +79,17 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.client.RestTemplate; import org.springframework.web.context.request.WebRequest; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; import org.springframework.web.util.UriComponentsBuilder; /** - * - * @author jejkal + * Controller for metadata documents. */ @Controller @RequestMapping(value = "/api/v1/metadata") @@ -107,16 +98,23 @@ public class MetadataControllerImpl implements IMetadataController { public static final String POST_FILTER = "post_filter"; - private static final String SID_READ = "read"; - final JsonNodeFactory factory = JsonNodeFactory.instance; + /** + * Placeholder string for id of resource. (landingpage) + */ + public static final String PLACEHOLDER_ID = "$(id)"; + /** + * Placeholder string for version of resource. (landingpage) + */ + public static final String PLACEHOLDER_VERSION = "$(version)"; private static final Logger LOG = LoggerFactory.getLogger(MetadataControllerImpl.class); - @Autowired + private final ApplicationProperties applicationProperties; + private final ILinkedMetadataRecordDao metadataRecordDao; private final MetastoreConfiguration metadataConfig; - @Autowired + private final IDataResourceDao dataResourceDao; /** @@ -130,16 +128,18 @@ public class MetadataControllerImpl implements IMetadataController { private final String guestToken; /** + * Constructor for metadata documents controller. * - * @param applicationProperties - * @param metadataConfig - * @param metadataRecordDao - * @param dataResourceDao + * @param applicationProperties Configuration for controller. + * @param metadataConfig Configuration for metadata documents repository. + * @param metadataRecordDao DAO for metadata records. + * @param dataResourceDao DAO for data resources. */ public MetadataControllerImpl(ApplicationProperties applicationProperties, MetastoreConfiguration metadataConfig, ILinkedMetadataRecordDao metadataRecordDao, IDataResourceDao dataResourceDao) { + this.applicationProperties = applicationProperties; this.metadataConfig = metadataConfig; this.metadataRecordDao = metadataRecordDao; this.dataResourceDao = dataResourceDao; @@ -165,14 +165,14 @@ public ResponseEntity createRecord( long nano1 = System.nanoTime() / 1000000; LOG.trace("Performing createRecord({},...).", recordDocument); - MetadataRecord record; + MetadataRecord metadataRecord; if (recordDocument == null || recordDocument.isEmpty()) { String message = "No metadata record provided. Returning HTTP BAD_REQUEST."; LOG.error(message); throw new BadArgumentException(message); } try { - record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); + metadataRecord = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord.class); } catch (IOException ex) { String message = "No valid metadata record provided. Returning HTTP BAD_REQUEST."; if (ex instanceof JsonParseException) { @@ -183,7 +183,7 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord } long nano2 = System.nanoTime() / 1000000; - if (record.getRelatedResource() == null || record.getRelatedResource().getIdentifier() == null || record.getSchema() == null || record.getSchema().getIdentifier() == null) { + if (metadataRecord.getRelatedResource() == null || metadataRecord.getRelatedResource().getIdentifier() == null || metadataRecord.getSchema() == null || metadataRecord.getSchema().getIdentifier() == null) { LOG.error("Mandatory attributes relatedResource and/or schemaId not found in record. Returning HTTP BAD_REQUEST."); return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Mandatory attributes relatedResource and/or schemaId not found in record."); } @@ -191,23 +191,27 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataRecord LOG.debug("Test for existing metadata record for given schema and resource"); ResourceIdentifier schemaIdentifier; try { - schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(metadataConfig, record); + schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(metadataConfig, metadataRecord); } catch (ResourceNotFoundException rnfe) { LOG.debug("Error checking for existing relations.", rnfe); throw new UnprocessableEntityException("Schema ID seems to be invalid"); } - boolean recordAlreadyExists = metadataRecordDao.existsMetadataRecordByRelatedResourceAndSchemaId(record.getRelatedResource().getIdentifier(), schemaIdentifier.getIdentifier()); + boolean recordAlreadyExists = metadataRecordDao.existsMetadataRecordByRelatedResourceAndSchemaId(metadataRecord.getRelatedResource().getIdentifier(), schemaIdentifier.getIdentifier()); long nano3 = System.nanoTime() / 1000000; if (recordAlreadyExists) { - LOG.error("Conflict with existing metadata record!"); - return ResponseEntity.status(HttpStatus.CONFLICT).body("Metadata record already exists! Please update existing record instead!"); + String message = String.format("Conflict! There is already a metadata document with " + + "the same schema ('%s') and the same related resource ('%s')", + metadataRecord.getSchemaId(), + metadataRecord.getRelatedResource().getIdentifier()); + LOG.error(message); + return ResponseEntity.status(HttpStatus.CONFLICT).body(message); } MetadataRecord result = MetadataRecordUtil.createMetadataRecord(metadataConfig, recordDocument, document); // Successfully created metadata record. long nano4 = System.nanoTime() / 1000000; LOG.trace("Metadata record successfully persisted. Returning result."); - fixMetadataDocumentUri(result); + MetadataRecordUtil.fixMetadataDocumentUri(result); long nano5 = System.nanoTime() / 1000000; metadataRecordDao.save(new LinkedMetadataRecord(result)); long nano6 = System.nanoTime() / 1000000; @@ -234,14 +238,14 @@ public ResponseEntity getRecordById( LOG.trace("Performing getRecordById({}, {}).", id, version); LOG.trace("Obtaining metadata record with id {} and version {}.", id, version); - MetadataRecord record = MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, version, true); + MetadataRecord metadataRecord = MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, version, true); LOG.trace("Metadata record found. Prepare response."); //if security enabled, check permission -> if not matching, return HTTP UNAUTHORIZED or FORBIDDEN LOG.trace("Get ETag of MetadataRecord."); - String etag = record.getEtag(); - fixMetadataDocumentUri(record); + String etag = metadataRecord.getEtag(); + MetadataRecordUtil.fixMetadataDocumentUri(metadataRecord); - return ResponseEntity.ok().eTag("\"" + etag + "\"").body(record); + return ResponseEntity.ok().eTag("\"" + etag + "\"").body(metadataRecord); } @Override @@ -256,11 +260,11 @@ public ResponseEntity getAclById( throw new AccessForbiddenException("Only for services!"); } - MetadataRecord record = MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, version, true); - fixMetadataDocumentUri(record); + MetadataRecord metadataRecord = MetadataRecordUtil.getRecordByIdAndVersion(metadataConfig, id, version, true); + MetadataRecordUtil.fixMetadataDocumentUri(metadataRecord); AclRecord aclRecord = new AclRecord(); - aclRecord.setAcl(record.getAcl()); - aclRecord.setMetadataRecord(record); + aclRecord.setAcl(metadataRecord.getAcl()); + aclRecord.setMetadataRecord(metadataRecord); return ResponseEntity.ok().body(aclRecord); } @@ -282,6 +286,26 @@ public ResponseEntity getMetadataDocumentById( body(new FileSystemResource(metadataDocumentPath.toFile())); } + @Override + public ModelAndView getLandingpageById( + @PathVariable(value = "id") String id, + @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr) { + LOG.trace("Performing Landing page for metadata document with ({}, {}).", id, version); + String redirectUrl = applicationProperties.getMetadataLandingPage(); + redirectUrl = redirectUrl.replace(PLACEHOLDER_ID, id); + String versionString = ""; + if (version != null) { + versionString = version.toString(); + } + redirectUrl = "redirect:" + redirectUrl.replace(PLACEHOLDER_VERSION, versionString); + + LOG.trace("Redirect to '{}'", redirectUrl); + + return new ModelAndView(redirectUrl); + } + public ResponseEntity> getAllVersions( @PathVariable(value = "id") String id, Pageable pgbl @@ -300,9 +324,9 @@ public ResponseEntity> getAllVersions( LOG.trace("Transforming Dataresource to MetadataRecord"); List metadataList = new ArrayList<>(); - recordList.forEach((record) -> { - fixMetadataDocumentUri(record); - metadataList.add(record); + recordList.forEach(metadataRecord -> { + MetadataRecordUtil.fixMetadataDocumentUri(metadataRecord); + metadataList.add(metadataRecord); }); String contentRange = ControllerUtils.getContentRangeHeader(pgbl.getPageNumber(), pgbl.getPageSize(), totalNoOfElements); @@ -344,7 +368,9 @@ public ResponseEntity> getRecords( } } } - List allRelatedIdentifiers = new ArrayList<>(); + List allRelatedIdentifiersSchema = new ArrayList<>(); + List allRelatedIdentifiersResource = new ArrayList<>(); + // File file = new File(new URIoa) if (schemaIds != null) { for (String schemaId : schemaIds) { @@ -353,40 +379,33 @@ public ResponseEntity> getRecords( currentSchemaRecord = MetadataRecordUtil.getCurrentInternalSchemaRecord(metadataConfig, schemaId); // Test for internal URI -> Transform to global URI. if (currentSchemaRecord.getSchemaDocumentUri().startsWith("file:")) { - ResourceIdentifier schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(metadataConfig, currentSchemaRecord); + ResourceIdentifier schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(currentSchemaRecord); currentSchemaRecord.setSchemaDocumentUri(schemaIdentifier.getIdentifier()); } - allRelatedIdentifiers.add(currentSchemaRecord.getSchemaDocumentUri()); - } catch (ResourceNotFoundException rnfe) { + allRelatedIdentifiersSchema.add(currentSchemaRecord.getSchemaDocumentUri()); + } catch (Exception rnfe) { // schemaID not found set version to 1 currentSchemaRecord = new MetadataSchemaRecord(); currentSchemaRecord.setSchemaVersion(1l); - allRelatedIdentifiers.add("UNKNOWN_SCHEMA_ID"); + allRelatedIdentifiersSchema.add("UNKNOWN_SCHEMA_ID"); } for (long versionNumber = 1; versionNumber < currentSchemaRecord.getSchemaVersion(); versionNumber++) { MetadataSchemaRecord schemaRecord = MetadataRecordUtil.getInternalSchemaRecord(metadataConfig, schemaId, versionNumber); // Test for internal URI -> Transform to global URI. if (schemaRecord.getSchemaDocumentUri().startsWith("file:")) { - ResourceIdentifier schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(metadataConfig, schemaRecord); + ResourceIdentifier schemaIdentifier = MetadataSchemaRecordUtil.getSchemaIdentifier(schemaRecord); schemaRecord.setSchemaDocumentUri(schemaIdentifier.getIdentifier()); } - allRelatedIdentifiers.add(schemaRecord.getSchemaDocumentUri()); + allRelatedIdentifiersSchema.add(schemaRecord.getSchemaDocumentUri()); } } + Specification schemaSpecification = RelatedIdentifierSpec.toSpecification(allRelatedIdentifiersSchema.toArray(new String[allRelatedIdentifiersSchema.size()])); + spec = spec.and(schemaSpecification); } if (relatedIds != null) { - allRelatedIdentifiers.addAll(relatedIds); - } - if (!allRelatedIdentifiers.isEmpty()) { - if (LOG.isTraceEnabled()) { - LOG.trace("---------------------------------------------------------"); - for (String relatedId : allRelatedIdentifiers) { - LOG.trace("Look for related Identifier: '{}'", relatedId); - } - LOG.trace("---------------------------------------------------------"); - } - Specification toSpecification = RelatedIdentifierSpec.toSpecification(allRelatedIdentifiers.toArray(new String[allRelatedIdentifiers.size()])); - spec = spec.and(toSpecification); + allRelatedIdentifiersResource.addAll(relatedIds); + Specification relResourceSpecification = RelatedIdentifierSpec.toSpecification(allRelatedIdentifiersResource.toArray(new String[allRelatedIdentifiersResource.size()])); + spec = spec.and(relResourceSpecification); } if ((updateFrom != null) || (updateUntil != null)) { spec = spec.and(LastUpdateSpecification.toSpecification(updateFrom, updateUntil)); @@ -413,9 +432,9 @@ public ResponseEntity> getRecords( LOG.trace("Transforming Dataresource to MetadataRecord"); List recordList = records.getContent(); List metadataList = new ArrayList<>(); - recordList.forEach((record) -> { - MetadataRecord item = MetadataRecordUtil.migrateToMetadataRecord(metadataConfig, record, false); - fixMetadataDocumentUri(item); + recordList.forEach(metadataRecord -> { + MetadataRecord item = MetadataRecordUtil.migrateToMetadataRecord(metadataConfig, metadataRecord, false); + MetadataRecordUtil.fixMetadataDocumentUri(item); metadataList.add(item); }); @@ -427,23 +446,21 @@ public ResponseEntity> getRecords( @Override public ResponseEntity updateRecord( @PathVariable("id") String id, - @RequestPart(name = "record", required = false) MultipartFile record, + @RequestPart(name = "record", required = false) MultipartFile metadataRecord, @RequestPart(name = "document", required = false) final MultipartFile document, WebRequest request, HttpServletResponse response, UriComponentsBuilder uriBuilder ) { - LOG.trace("Performing updateRecord({}, {}, {}).", id, record, "#document"); - Function getById; - getById = (t) -> { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, response)).toString(); - }; + LOG.trace("Performing updateRecord({}, {}, {}).", id, metadataRecord, "#document"); + UnaryOperator getById; + getById = t -> WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, response)).toString(); String eTag = ControllerUtils.getEtagFromHeader(request); - MetadataRecord updateMetadataRecord = MetadataRecordUtil.updateMetadataRecord(metadataConfig, id, eTag, record, document, getById); + MetadataRecord updateMetadataRecord = MetadataRecordUtil.updateMetadataRecord(metadataConfig, id, eTag, metadataRecord, document, getById); LOG.trace("Metadata record successfully persisted. Updating document URI and returning result."); String etag = updateMetadataRecord.getEtag(); - fixMetadataDocumentUri(updateMetadataRecord); + MetadataRecordUtil.fixMetadataDocumentUri(updateMetadataRecord); URI locationUri; locationUri = WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(updateMetadataRecord.getId(), updateMetadataRecord.getRecordVersion(), null, null)).toUri(); @@ -462,10 +479,9 @@ public ResponseEntity deleteRecord( HttpServletResponse hsr ) { LOG.trace("Performing deleteRecord({}).", id); - Function getById; - getById = (t) -> { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, wr, hsr)).toString(); - }; + UnaryOperator getById; + getById = t -> WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, wr, hsr)).toString(); + String eTag = ControllerUtils.getEtagFromHeader(wr); MetadataRecordUtil.deleteMetadataRecord(metadataConfig, id, eTag, getById); @@ -484,15 +500,9 @@ public void contribute(Info.Builder builder) { builder.withDetail("metadataRepo", details); } } - + @Bean public RestTemplate restTemplate() { return new RestTemplate(); } - - private void fixMetadataDocumentUri(MetadataRecord record) { - String metadataDocumentUri = record.getMetadataDocumentUri(); - record.setMetadataDocumentUri(WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getMetadataDocumentById(record.getId(), record.getRecordVersion(), null, null)).toUri().toString()); - LOG.trace("Fix metadata document Uri '{}' -> '{}'", metadataDocumentUri, record.getMetadataDocumentUri()); - } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataEditorController.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataEditorController.java index 93367a7b..7f9f3301 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataEditorController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataEditorController.java @@ -14,7 +14,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import javax.servlet.http.HttpServletResponse; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.slf4j.Logger; @@ -28,8 +27,11 @@ import org.springframework.web.servlet.ModelAndView; import org.springframework.web.util.UriComponentsBuilder; import edu.kit.datamanager.metastore2.web.IMetadataEditorController; +import jakarta.servlet.http.HttpServletResponse; +import org.json.simple.parser.ParseException; /** + * Controller for the metadata editor web frontend. * * @author sabrinechelbi */ @@ -41,13 +43,13 @@ public class MetadataEditorController implements IMetadataEditorController { private static final Logger LOG = LoggerFactory.getLogger(MetadataEditorController.class); - private final static String DATAMODELSCHEMA = "/static/jsonSchemas/schemaRecord.json"; - private final static String UIFORMSCHEMA = "/static/jsonSchemas/uiFormSchemaRecord.json"; - private final static String ITEMSSCHEMA = "/static/jsonSchemas/itemsSchemaRecord.json"; - - private final static String DATAMODELMETADATA = "/static/jsonSchemas/metadataRecord.json"; - private final static String UIFORMMETADATA = "/static/jsonSchemas/uiFormMetadataRecord.json"; - private final static String ITEMSMETADATA = "/static/jsonSchemas/itemsMetadataRecord.json"; + private static final String DATAMODELSCHEMA = "/static/jsonSchemas/schemaRecord.json"; + private static final String UIFORMSCHEMA = "/static/jsonSchemas/uiFormSchemaRecord.json"; + private static final String ITEMSSCHEMA = "/static/jsonSchemas/itemsSchemaRecord.json"; + + private static final String DATAMODELMETADATA = "/static/jsonSchemas/metadataRecord.json"; + private static final String UIFORMMETADATA = "/static/jsonSchemas/uiFormMetadataRecord.json"; + private static final String ITEMSMETADATA = "/static/jsonSchemas/itemsMetadataRecord.json"; @Override public ModelAndView schemaManagement() { @@ -95,9 +97,9 @@ private JSONObject getJsonObject(String path) { JSONObject obj = null; try { obj = (JSONObject) parser.parse( - new InputStreamReader(resource.getInputStream(), "UTF-8")); - } catch (Exception e) { - e.printStackTrace(); + new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8)); + } catch (IOException | ParseException e) { + LOG.error("Error parsing JSON object!", e); } return obj; } diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataSearchController.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataSearchController.java index 5e9b443b..7c61ba39 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataSearchController.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/MetadataSearchController.java @@ -49,16 +49,16 @@ public class MetadataSearchController { private static final Logger LOG = LoggerFactory.getLogger(MetadataSearchController.class); + + private static final String SEARCH_PATH_POSTFIX = "/_search"; @Autowired private SearchConfiguration searchConfiguration; /** + * Constructor with configuration. * - * @param applicationProperties - * @param metadataConfig - * @param metadataRecordDao - * @param dataResourceDao + * @param searchConfiguration Configuration for search. */ public MetadataSearchController(SearchConfiguration searchConfiguration) { this.searchConfiguration = searchConfiguration; @@ -92,13 +92,14 @@ public ResponseEntity proxy(@RequestBody JsonNode body, + "to which the records refer as comma-separated values. " + "Regular expressions are also allowed. " + "See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/multi-index.html", required = true) @PathVariable(value = "schemaId") String schemaIds, - ProxyExchange proxy, + ProxyExchange proxy, @Parameter(hidden = true) final Pageable pgbl) throws Exception { // Prepare query with authorization - ObjectNode on = prepareQuery(body, pgbl); + prepareQuery(body, pgbl); + LOG.trace("Redirect post to " + searchConfiguration.getUrl() + "/" + schemaIds + SEARCH_PATH_POSTFIX); - return proxy.uri(searchConfiguration.getUrl() + "/" + schemaIds + "/_search").post(); + return proxy.uri(searchConfiguration.getUrl() + "/" + schemaIds + SEARCH_PATH_POSTFIX).post(); } @PostMapping("/search") @@ -122,13 +123,14 @@ public ResponseEntity proxy(@RequestBody JsonNode body, @ResponseBody @PageableAsQueryParam public ResponseEntity proxy(@RequestBody JsonNode body, - ProxyExchange proxy, + ProxyExchange proxy, @Parameter(hidden = true) final Pageable pgbl) throws Exception { // Prepare query with authorization - ObjectNode on = prepareQuery(body, pgbl); + prepareQuery(body, pgbl); + LOG.trace("Redirect post to " + searchConfiguration.getUrl() + SEARCH_PATH_POSTFIX); - return proxy.uri(searchConfiguration.getUrl() + "/_search").post(); + return proxy.uri(searchConfiguration.getUrl() + SEARCH_PATH_POSTFIX).post(); } /** diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/SchemaRegistryControllerImpl.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/SchemaRegistryControllerImpl.java index 7da2a812..f40d708a 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/SchemaRegistryControllerImpl.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/SchemaRegistryControllerImpl.java @@ -17,10 +17,10 @@ import edu.kit.datamanager.entities.PERMISSION; import edu.kit.datamanager.entities.RepoUserRole; +import edu.kit.datamanager.exceptions.ResourceNotFoundException; +import edu.kit.datamanager.metastore2.configuration.ApplicationProperties; import edu.kit.datamanager.metastore2.configuration.MetastoreConfiguration; -import edu.kit.datamanager.metastore2.dao.IUrl2PathDao; import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; -import edu.kit.datamanager.metastore2.domain.Url2Path; import edu.kit.datamanager.metastore2.util.ActuatorUtil; import edu.kit.datamanager.metastore2.util.MetadataSchemaRecordUtil; import edu.kit.datamanager.metastore2.web.ISchemaRegistryController; @@ -36,6 +36,8 @@ import edu.kit.datamanager.util.ControllerUtils; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.net.URI; import java.net.URL; import java.nio.file.Files; @@ -47,12 +49,9 @@ import java.util.List; import java.util.Map; import java.util.function.BiFunction; -import java.util.function.Function; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import java.util.function.UnaryOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.info.Info; import org.springframework.core.io.FileSystemResource; import org.springframework.data.domain.Page; @@ -70,11 +69,11 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.context.request.WebRequest; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; import org.springframework.web.util.UriComponentsBuilder; /** - * - * @author jejkal + * Controller for schema documents. */ @Controller @RequestMapping(value = "/api/v1/schemas") @@ -84,25 +83,25 @@ public class SchemaRegistryControllerImpl implements ISchemaRegistryController { private static final Logger LOG = LoggerFactory.getLogger(SchemaRegistryControllerImpl.class); - @Autowired + private final ApplicationProperties applicationProperties; + private final MetastoreConfiguration schemaConfig; - @Autowired + private final IDataResourceDao dataResourceDao; - @Autowired - private final IUrl2PathDao url2PathDao; /** + * Constructor for schema documents controller. * - * @param schemaConfig - * @param dataResourceDao - * @param contentInformationDao + * @param applicationProperties Configuration for controller. + * @param schemaConfig Configuration for metadata documents repository. + * @param dataResourceDao DAO for data resources. */ - public SchemaRegistryControllerImpl(MetastoreConfiguration schemaConfig, - IDataResourceDao dataResourceDao, - IUrl2PathDao url2PathDao) { + public SchemaRegistryControllerImpl(ApplicationProperties applicationProperties, + MetastoreConfiguration schemaConfig, + IDataResourceDao dataResourceDao) { + this.applicationProperties = applicationProperties; this.schemaConfig = schemaConfig; this.dataResourceDao = dataResourceDao; - this.url2PathDao = url2PathDao; LOG.info("------------------------------------------------------"); LOG.info("------{}", schemaConfig); LOG.info("------------------------------------------------------"); @@ -117,19 +116,18 @@ public ResponseEntity createRecord( UriComponentsBuilder uriBuilder) { LOG.trace("Performing createRecord({},....", recordDocument); BiFunction getSchemaDocumentById; - getSchemaDocumentById = (schema, version) -> { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getSchemaDocumentById(schema, version, null, null)).toString(); - }; - MetadataSchemaRecord record = MetadataSchemaRecordUtil.createMetadataSchemaRecord(schemaConfig, recordDocument, document, getSchemaDocumentById); + getSchemaDocumentById = (schema, version) -> WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getSchemaDocumentById(schema, version, null, null)).toString(); + + MetadataSchemaRecord schemaRecord = MetadataSchemaRecordUtil.createMetadataSchemaRecord(schemaConfig, recordDocument, document, getSchemaDocumentById); LOG.trace("Schema record successfully persisted. Returning result."); - String etag = record.getEtag(); + String etag = schemaRecord.getEtag(); LOG.trace("Schema record successfully persisted. Updating document URI."); - fixSchemaDocumentUri(record, true); + MetadataSchemaRecordUtil.fixSchemaDocumentUri(schemaRecord, true); URI locationUri; - locationUri = getSchemaDocumentUri(record); + locationUri = MetadataSchemaRecordUtil.getSchemaDocumentUri(schemaRecord); LOG.warn("location uri " + locationUri); - return ResponseEntity.created(locationUri).eTag("\"" + etag + "\"").body(record); + return ResponseEntity.created(locationUri).eTag("\"" + etag + "\"").body(schemaRecord); } @Override @@ -141,12 +139,31 @@ public ResponseEntity getRecordById( LOG.trace("Performing getRecordById({}, {}).", schemaId, version); LOG.trace("Obtaining schema record with id {} and version {}.", schemaId, version); - MetadataSchemaRecord record = MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, schemaId, version, true); - String etag = record.getEtag(); + MetadataSchemaRecord schemaRecord = MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, schemaId, version, true); + String etag = schemaRecord.getEtag(); - fixSchemaDocumentUri(record); + MetadataSchemaRecordUtil.fixSchemaDocumentUri(schemaRecord); LOG.trace("Document URI successfully updated. Returning result."); - return ResponseEntity.ok().eTag("\"" + etag + "\"").body(record); + return ResponseEntity.ok().eTag("\"" + etag + "\"").body(schemaRecord); + } + + @Override + public ModelAndView getLandingPageById(@PathVariable(value = "schemaId") String id, + @RequestParam(value = "version", required = false) Long version, + WebRequest wr, + HttpServletResponse hsr) { + LOG.trace("Performing Landing page for schema document with ({}, {}).", id, version); + String redirectUrl = applicationProperties.getSchemaLandingPage(); + redirectUrl = redirectUrl.replace(MetadataControllerImpl.PLACEHOLDER_ID, id); + String versionString = ""; + if (version != null) { + versionString = version.toString(); + } + redirectUrl = "redirect:" + redirectUrl.replace(MetadataControllerImpl.PLACEHOLDER_VERSION, versionString); + + LOG.trace("Redirect to '{}'", redirectUrl); + + return new ModelAndView(redirectUrl); } @Override @@ -158,10 +175,10 @@ public ResponseEntity getSchemaDocumentById( LOG.trace("Performing getSchemaDocumentById({}, {}).", schemaId, version); LOG.trace("Obtaining schema record with id {} and version {}.", schemaId, version); - MetadataSchemaRecord record = MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, schemaId, version); - URI schemaDocumentUri = URI.create(record.getSchemaDocumentUri()); + MetadataSchemaRecord schemaRecord = MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, schemaId, version); + URI schemaDocumentUri = URI.create(schemaRecord.getSchemaDocumentUri()); - MediaType contentType = MetadataSchemaRecord.SCHEMA_TYPE.XML.equals(record.getType()) ? MediaType.APPLICATION_XML : MediaType.APPLICATION_JSON; + MediaType contentType = MetadataSchemaRecord.SCHEMA_TYPE.XML.equals(schemaRecord.getType()) ? MediaType.APPLICATION_XML : MediaType.APPLICATION_JSON; Path schemaDocumentPath = Paths.get(schemaDocumentUri); if (!Files.exists(schemaDocumentPath) || !Files.isRegularFile(schemaDocumentPath) || !Files.isReadable(schemaDocumentPath)) { LOG.trace("Schema document at path {} either does not exist or is no file or is not readable. Returning HTTP NOT_FOUND.", schemaDocumentPath); @@ -184,18 +201,24 @@ public ResponseEntity> getAllVersions( //if security is enabled, include principal in query LOG.debug("Performing query for records."); - MetadataSchemaRecord recordByIdAndVersion = MetadataSchemaRecordUtil.getRecordById(schemaConfig, id); + MetadataSchemaRecord recordByIdAndVersion = null; List recordList = new ArrayList<>(); - long totalNoOfElements = recordByIdAndVersion.getSchemaVersion(); - for (long version = totalNoOfElements - pgbl.getOffset(), size = 0; version > 0 && size < pgbl.getPageSize(); version--, size++) { - recordList.add(MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, id, version)); + long totalNoOfElements = 5; + try { + recordByIdAndVersion = MetadataSchemaRecordUtil.getRecordById(schemaConfig, id); + totalNoOfElements = recordByIdAndVersion.getSchemaVersion(); + for (long version = totalNoOfElements - pgbl.getOffset(), size = 0; version > 0 && size < pgbl.getPageSize(); version--, size++) { + recordList.add(MetadataSchemaRecordUtil.getRecordByIdAndVersion(schemaConfig, id, version)); + } + } catch (ResourceNotFoundException rnfe) { + LOG.info("Schema ID '{}' is unkown. Return empty list...", id); } LOG.trace("Transforming Dataresource to MetadataRecord"); List metadataList = new ArrayList<>(); - recordList.forEach((record) -> { - fixSchemaDocumentUri(record); - metadataList.add(record); + recordList.forEach(schemaRecord -> { + MetadataSchemaRecordUtil.fixSchemaDocumentUri(schemaRecord); + metadataList.add(schemaRecord); }); String contentRange = ControllerUtils.getContentRangeHeader(pgbl.getPageNumber(), pgbl.getPageSize(), totalNoOfElements); @@ -205,9 +228,9 @@ public ResponseEntity> getAllVersions( @Override public ResponseEntity validate(@PathVariable(value = "schemaId") String schemaId, - @RequestParam(value = "version", required = false) Long version, - MultipartFile document, - WebRequest wr, + @RequestParam(value = "version", required = false) Long version, + MultipartFile document, + WebRequest wr, HttpServletResponse hsr) { LOG.trace("Performing validate({}, {}, {}).", schemaId, version, "#document"); MetadataSchemaRecordUtil.validateMetadataDocument(schemaConfig, document, schemaId, version); @@ -216,13 +239,13 @@ public ResponseEntity validate(@PathVariable(value = "schemaId") String schemaId } @Override - public ResponseEntity> getRecords(@RequestParam(value = "schemaId", required = false) String schemaId, + public ResponseEntity> getRecords(@RequestParam(value = "schemaId", required = false) String schemaId, @RequestParam(value = "mimeType", required = false) List mimeTypes, @RequestParam(name = "from", required = false) Instant updateFrom, @RequestParam(name = "until", required = false) Instant updateUntil, - Pageable pgbl, - WebRequest wr, - HttpServletResponse hsr, + Pageable pgbl, + WebRequest wr, + HttpServletResponse hsr, UriComponentsBuilder ucb) { LOG.trace("Performing getRecords({}, {}, {}, {}).", schemaId, mimeTypes, updateFrom, updateUntil); // if schemaId is given return all versions @@ -232,21 +255,7 @@ public ResponseEntity> getRecords(@RequestParam(value // Search for resource type of MetadataSchemaRecord Specification spec = ResourceTypeSpec.toSpecification(ResourceType.createResourceType(MetadataSchemaRecord.RESOURCE_TYPE)); // Add authentication if enabled - if (schemaConfig.isAuthEnabled()) { - boolean isAdmin; - isAdmin = AuthenticationHelper.hasAuthority(RepoUserRole.ADMINISTRATOR.toString()); - // Add authorization for non administrators - if (!isAdmin) { - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification."); - Specification permissionSpec = PermissionSpecification.toSpecification(authorizationIdentities, PERMISSION.READ); - spec = spec.and(permissionSpec); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); - } - } - } + spec = addAuthenticationSpecification(spec); //one of given mimetypes. if ((mimeTypes != null) && !mimeTypes.isEmpty()) { spec = spec.and(TitleSpec.toSpecification(mimeTypes.toArray(new String[mimeTypes.size()]))); @@ -275,9 +284,9 @@ public ResponseEntity> getRecords(@RequestParam(value } } List schemaList = new ArrayList<>(); - recordList.forEach((record) -> { - MetadataSchemaRecord item = MetadataSchemaRecordUtil.migrateToMetadataSchemaRecord(schemaConfig, record, false); - fixSchemaDocumentUri(item); + recordList.forEach(schemaRecord -> { + MetadataSchemaRecord item = MetadataSchemaRecordUtil.migrateToMetadataSchemaRecord(schemaConfig, schemaRecord, false); + MetadataSchemaRecordUtil.fixSchemaDocumentUri(item); schemaList.add(item); if (LOG.isTraceEnabled()) { LOG.trace("===> " + item.toString()); @@ -291,39 +300,36 @@ public ResponseEntity> getRecords(@RequestParam(value @Override public ResponseEntity updateRecord(@PathVariable("schemaId") final String schemaId, - @RequestPart(name = "record", required = false) MultipartFile record, + @RequestPart(name = "record", required = false) MultipartFile schemaRecord, @RequestPart(name = "schema", required = false) final MultipartFile document, final WebRequest request, final HttpServletResponse response) { - LOG.trace("Performing updateRecord({}, {}).", schemaId, record); - Function getById; - getById = (t) -> { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, response)).toString(); - }; + LOG.trace("Performing updateRecord({}, {}).", schemaId, schemaRecord); + UnaryOperator getById; + getById = t -> WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, response)).toString(); String eTag = ControllerUtils.getEtagFromHeader(request); - MetadataSchemaRecord updatedSchemaRecord = MetadataSchemaRecordUtil.updateMetadataSchemaRecord(schemaConfig, schemaId, eTag, record, document, getById); + MetadataSchemaRecord updatedSchemaRecord = MetadataSchemaRecordUtil.updateMetadataSchemaRecord(schemaConfig, schemaId, eTag, schemaRecord, document, getById); LOG.trace("Metadata record successfully persisted. Updating document URI and returning result."); String etag = updatedSchemaRecord.getEtag(); - fixSchemaDocumentUri(updatedSchemaRecord, true); + MetadataSchemaRecordUtil.fixSchemaDocumentUri(updatedSchemaRecord, true); // Fix Url for OAI PMH entry MetadataSchemaRecordUtil.updateMetadataFormat(updatedSchemaRecord); URI locationUri; - locationUri = getSchemaDocumentUri(updatedSchemaRecord); + locationUri = MetadataSchemaRecordUtil.getSchemaDocumentUri(updatedSchemaRecord); LOG.trace("Set locationUri to '{}'", locationUri.toString()); return ResponseEntity.ok().location(locationUri).eTag("\"" + etag + "\"").body(updatedSchemaRecord); } @Override public ResponseEntity deleteRecord(@PathVariable("schemaId") final String schemaId, - WebRequest request, + WebRequest request, HttpServletResponse hsr) { LOG.trace("Performing deleteRecord({}).", schemaId); - Function getById; - getById = (t) -> { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, hsr)).toString(); - }; + UnaryOperator getById; + getById = t -> WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getRecordById(t, null, request, hsr)).toString(); String eTag = ControllerUtils.getEtagFromHeader(request); + MetadataSchemaRecordUtil.deleteMetadataSchemaRecord(schemaConfig, schemaId, eTag, getById); return new ResponseEntity<>(HttpStatus.NO_CONTENT); @@ -342,43 +348,22 @@ public void contribute(Info.Builder builder) { } } - /** - * Fix local document URI to URL. - * - * @param record record holding schemaId and version of local document. - */ - private void fixSchemaDocumentUri(MetadataSchemaRecord record) { - fixSchemaDocumentUri(record, false); - } - - /** - * Fix local document URI to URL. - * - * @param record record holding schemaId and version of local document. - * @param saveUrl save path to file for URL. - */ - private void fixSchemaDocumentUri(MetadataSchemaRecord record, boolean saveUrl) { - String schemaDocumentUri = record.getSchemaDocumentUri(); - record.setSchemaDocumentUri(getSchemaDocumentUri(record).toString()); - LOG.trace("Fix schema document Uri '{}' -> '{}'", schemaDocumentUri, record.getSchemaDocumentUri()); - if (saveUrl) { - LOG.trace("Store path for URI!"); - Url2Path url2Path = new Url2Path(); - url2Path.setPath(schemaDocumentUri); - url2Path.setUrl(record.getSchemaDocumentUri()); - url2Path.setType(record.getType()); - url2Path.setVersion(record.getSchemaVersion()); - url2PathDao.save(url2Path); + private Specification addAuthenticationSpecification(Specification spec) { + if (schemaConfig.isAuthEnabled()) { + boolean isAdmin; + isAdmin = AuthenticationHelper.hasAuthority(RepoUserRole.ADMINISTRATOR.toString()); + // Add authorization for non administrators + if (!isAdmin) { + List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); + if (authorizationIdentities != null) { + LOG.trace("Creating (READ) permission specification."); + Specification permissionSpec = PermissionSpecification.toSpecification(authorizationIdentities, PERMISSION.READ); + spec = spec.and(permissionSpec); + } else { + LOG.trace("No permission information provided. Skip creating permission specification."); + } + } } - } - - /** - * Get URI for accessing schema document via schemaId and version. - * - * @param record Record holding schemaId and version. - * @return URI for accessing schema document. - */ - public URI getSchemaDocumentUri(MetadataSchemaRecord record) { - return WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(this.getClass()).getSchemaDocumentById(record.getSchemaId(), record.getSchemaVersion(), null, null)).toUri(); + return spec; } } diff --git a/src/main/java/org/openarchives/oai/_2/AboutType.java b/src/main/java/org/openarchives/oai/_2/AboutType.java index 442628e2..849801d6 100644 --- a/src/main/java/org/openarchives/oai/_2/AboutType.java +++ b/src/main/java/org/openarchives/oai/_2/AboutType.java @@ -8,10 +8,14 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAnyElement; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAnyElement; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAnyElement; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/DeletedRecordType.java b/src/main/java/org/openarchives/oai/_2/DeletedRecordType.java index 42b3ae41..a3693d6b 100644 --- a/src/main/java/org/openarchives/oai/_2/DeletedRecordType.java +++ b/src/main/java/org/openarchives/oai/_2/DeletedRecordType.java @@ -8,9 +8,9 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/DescriptionType.java b/src/main/java/org/openarchives/oai/_2/DescriptionType.java index 144c1471..f3c509d9 100644 --- a/src/main/java/org/openarchives/oai/_2/DescriptionType.java +++ b/src/main/java/org/openarchives/oai/_2/DescriptionType.java @@ -8,10 +8,10 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAnyElement; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAnyElement; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/GetRecordType.java b/src/main/java/org/openarchives/oai/_2/GetRecordType.java index 62f2dd68..88805c39 100644 --- a/src/main/java/org/openarchives/oai/_2/GetRecordType.java +++ b/src/main/java/org/openarchives/oai/_2/GetRecordType.java @@ -8,10 +8,10 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/GranularityType.java b/src/main/java/org/openarchives/oai/_2/GranularityType.java index fe5423bf..e4cb9e05 100644 --- a/src/main/java/org/openarchives/oai/_2/GranularityType.java +++ b/src/main/java/org/openarchives/oai/_2/GranularityType.java @@ -8,9 +8,9 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/HeaderType.java b/src/main/java/org/openarchives/oai/_2/HeaderType.java index 51153cb4..a09c0e11 100644 --- a/src/main/java/org/openarchives/oai/_2/HeaderType.java +++ b/src/main/java/org/openarchives/oai/_2/HeaderType.java @@ -8,14 +8,14 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; /** @@ -136,7 +136,7 @@ public void setDatestamp(String value) { */ public List getSetSpec() { if (setSpec == null) { - setSpec = new ArrayList(); + setSpec = new ArrayList<>(); } return this.setSpec; } diff --git a/src/main/java/org/openarchives/oai/_2/IdentifyType.java b/src/main/java/org/openarchives/oai/_2/IdentifyType.java index 1b633f6d..4f447d22 100644 --- a/src/main/java/org/openarchives/oai/_2/IdentifyType.java +++ b/src/main/java/org/openarchives/oai/_2/IdentifyType.java @@ -8,13 +8,13 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; /** @@ -175,7 +175,7 @@ public void setProtocolVersion(String value) { */ public List getAdminEmail() { if (adminEmail == null) { - adminEmail = new ArrayList(); + adminEmail = new ArrayList<>(); } return this.adminEmail; } @@ -276,7 +276,7 @@ public void setGranularity(GranularityType value) { */ public List getCompression() { if (compression == null) { - compression = new ArrayList(); + compression = new ArrayList<>(); } return this.compression; } @@ -305,7 +305,7 @@ public List getCompression() { */ public List getDescription() { if (description == null) { - description = new ArrayList(); + description = new ArrayList<>(); } return this.description; } diff --git a/src/main/java/org/openarchives/oai/_2/ListIdentifiersType.java b/src/main/java/org/openarchives/oai/_2/ListIdentifiersType.java index 7db40c7d..28c382a1 100644 --- a/src/main/java/org/openarchives/oai/_2/ListIdentifiersType.java +++ b/src/main/java/org/openarchives/oai/_2/ListIdentifiersType.java @@ -8,12 +8,12 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; /** @@ -71,7 +71,7 @@ public class ListIdentifiersType { */ public List getHeader() { if (header == null) { - header = new ArrayList(); + header = new ArrayList<>(); } return this.header; } diff --git a/src/main/java/org/openarchives/oai/_2/ListMetadataFormatsType.java b/src/main/java/org/openarchives/oai/_2/ListMetadataFormatsType.java index d13e63c5..4914b1d3 100644 --- a/src/main/java/org/openarchives/oai/_2/ListMetadataFormatsType.java +++ b/src/main/java/org/openarchives/oai/_2/ListMetadataFormatsType.java @@ -6,12 +6,12 @@ // package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; /** *

@@ -68,7 +68,7 @@ public class ListMetadataFormatsType { */ public List getMetadataFormat() { if (metadataFormat == null) { - metadataFormat = new ArrayList(); + metadataFormat = new ArrayList<>(); } return this.metadataFormat; } diff --git a/src/main/java/org/openarchives/oai/_2/ListRecordsType.java b/src/main/java/org/openarchives/oai/_2/ListRecordsType.java index c52dc669..ae271662 100644 --- a/src/main/java/org/openarchives/oai/_2/ListRecordsType.java +++ b/src/main/java/org/openarchives/oai/_2/ListRecordsType.java @@ -8,12 +8,12 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; /** @@ -71,7 +71,7 @@ public class ListRecordsType { */ public List getRecord() { if (record == null) { - record = new ArrayList(); + record = new ArrayList<>(); } return this.record; } diff --git a/src/main/java/org/openarchives/oai/_2/ListSetsType.java b/src/main/java/org/openarchives/oai/_2/ListSetsType.java index 78662eb6..2f6cbecb 100644 --- a/src/main/java/org/openarchives/oai/_2/ListSetsType.java +++ b/src/main/java/org/openarchives/oai/_2/ListSetsType.java @@ -8,12 +8,12 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; /** @@ -71,7 +71,7 @@ public class ListSetsType { */ public List getSet() { if (set == null) { - set = new ArrayList(); + set = new ArrayList<>(); } return this.set; } diff --git a/src/main/java/org/openarchives/oai/_2/MetadataFormatType.java b/src/main/java/org/openarchives/oai/_2/MetadataFormatType.java index f6ace0e5..3e1e5fc4 100644 --- a/src/main/java/org/openarchives/oai/_2/MetadataFormatType.java +++ b/src/main/java/org/openarchives/oai/_2/MetadataFormatType.java @@ -8,11 +8,11 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/MetadataType.java b/src/main/java/org/openarchives/oai/_2/MetadataType.java index 78816048..eb759091 100644 --- a/src/main/java/org/openarchives/oai/_2/MetadataType.java +++ b/src/main/java/org/openarchives/oai/_2/MetadataType.java @@ -6,10 +6,10 @@ // package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAnyElement; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAnyElement; +import jakarta.xml.bind.annotation.XmlType; /** * Metadata must be expressed in XML that complies with another XML Schema diff --git a/src/main/java/org/openarchives/oai/_2/OAIPMHerrorType.java b/src/main/java/org/openarchives/oai/_2/OAIPMHerrorType.java index 715018c6..c33d67de 100644 --- a/src/main/java/org/openarchives/oai/_2/OAIPMHerrorType.java +++ b/src/main/java/org/openarchives/oai/_2/OAIPMHerrorType.java @@ -8,11 +8,16 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlType; -import javax.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; /** diff --git a/src/main/java/org/openarchives/oai/_2/OAIPMHerrorcodeType.java b/src/main/java/org/openarchives/oai/_2/OAIPMHerrorcodeType.java index 814d87f4..ea478785 100644 --- a/src/main/java/org/openarchives/oai/_2/OAIPMHerrorcodeType.java +++ b/src/main/java/org/openarchives/oai/_2/OAIPMHerrorcodeType.java @@ -8,9 +8,9 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/OAIPMHtype.java b/src/main/java/org/openarchives/oai/_2/OAIPMHtype.java index c8cd62d5..3b6a1041 100644 --- a/src/main/java/org/openarchives/oai/_2/OAIPMHtype.java +++ b/src/main/java/org/openarchives/oai/_2/OAIPMHtype.java @@ -6,15 +6,15 @@ // package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; import javax.xml.datatype.XMLGregorianCalendar; /** @@ -143,11 +143,11 @@ public void setRequest(RequestType value){ * Objects of the following type(s) are allowed in the list * {@link OAIPMHerrorType } * - * + * @return List containing all detected errors. */ public List getError(){ if(error == null){ - error = new ArrayList(); + error = new ArrayList<>(); } return this.error; } diff --git a/src/main/java/org/openarchives/oai/_2/ObjectFactory.java b/src/main/java/org/openarchives/oai/_2/ObjectFactory.java index c9cba605..f9fa3874 100644 --- a/src/main/java/org/openarchives/oai/_2/ObjectFactory.java +++ b/src/main/java/org/openarchives/oai/_2/ObjectFactory.java @@ -8,9 +8,9 @@ package org.openarchives.oai._2; -import javax.xml.bind.JAXBElement; -import javax.xml.bind.annotation.XmlElementDecl; -import javax.xml.bind.annotation.XmlRegistry; +import jakarta.xml.bind.JAXBElement; +import jakarta.xml.bind.annotation.XmlElementDecl; +import jakarta.xml.bind.annotation.XmlRegistry; import javax.xml.namespace.QName; @@ -31,13 +31,14 @@ @XmlRegistry public class ObjectFactory { - private final static QName _OAIPMH_QNAME = new QName("http://www.openarchives.org/OAI/2.0/", "OAI-PMH"); + private static final QName _OAIPMH_QNAME = new QName("http://www.openarchives.org/OAI/2.0/", "OAI-PMH"); /** * Create a new ObjectFactory that can be used to create new instances of schema derived classes for package: org.openarchives.oai._2 * */ public ObjectFactory() { + // Generated constructor. } /** @@ -198,7 +199,7 @@ public MetadataType createMetadataType() { */ @XmlElementDecl(namespace = "http://www.openarchives.org/OAI/2.0/", name = "OAI-PMH") public JAXBElement createOAIPMH(OAIPMHtype value) { - return new JAXBElement(_OAIPMH_QNAME, OAIPMHtype.class, null, value); + return new JAXBElement<>(_OAIPMH_QNAME, OAIPMHtype.class, null, value); } } diff --git a/src/main/java/org/openarchives/oai/_2/RecordType.java b/src/main/java/org/openarchives/oai/_2/RecordType.java index 0715cf7b..9c9b4bea 100644 --- a/src/main/java/org/openarchives/oai/_2/RecordType.java +++ b/src/main/java/org/openarchives/oai/_2/RecordType.java @@ -8,12 +8,12 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; /** @@ -125,7 +125,7 @@ public void setMetadata(MetadataType value) { */ public List getAbout() { if (about == null) { - about = new ArrayList(); + about = new ArrayList<>(); } return this.about; } diff --git a/src/main/java/org/openarchives/oai/_2/RequestType.java b/src/main/java/org/openarchives/oai/_2/RequestType.java index 410e75ad..5dfda671 100644 --- a/src/main/java/org/openarchives/oai/_2/RequestType.java +++ b/src/main/java/org/openarchives/oai/_2/RequestType.java @@ -8,12 +8,12 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; -import javax.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; /** diff --git a/src/main/java/org/openarchives/oai/_2/ResumptionTokenType.java b/src/main/java/org/openarchives/oai/_2/ResumptionTokenType.java index 14b2f4fa..a793ba34 100644 --- a/src/main/java/org/openarchives/oai/_2/ResumptionTokenType.java +++ b/src/main/java/org/openarchives/oai/_2/ResumptionTokenType.java @@ -8,13 +8,13 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; import java.math.BigInteger; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlSchemaType; -import javax.xml.bind.annotation.XmlType; -import javax.xml.bind.annotation.XmlValue; import javax.xml.datatype.XMLGregorianCalendar; diff --git a/src/main/java/org/openarchives/oai/_2/SetType.java b/src/main/java/org/openarchives/oai/_2/SetType.java index c6d396ad..e27f8068 100644 --- a/src/main/java/org/openarchives/oai/_2/SetType.java +++ b/src/main/java/org/openarchives/oai/_2/SetType.java @@ -8,12 +8,16 @@ package org.openarchives.oai._2; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; import java.util.ArrayList; import java.util.List; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlType; /** @@ -123,7 +127,7 @@ public void setSetName(String value) { */ public List getSetDescription() { if (setDescription == null) { - setDescription = new ArrayList(); + setDescription = new ArrayList<>(); } return this.setDescription; } diff --git a/src/main/java/org/openarchives/oai/_2/StatusType.java b/src/main/java/org/openarchives/oai/_2/StatusType.java index be39d173..5eb541d2 100644 --- a/src/main/java/org/openarchives/oai/_2/StatusType.java +++ b/src/main/java/org/openarchives/oai/_2/StatusType.java @@ -8,9 +8,12 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/VerbType.java b/src/main/java/org/openarchives/oai/_2/VerbType.java index cae98b8e..a21a3f05 100644 --- a/src/main/java/org/openarchives/oai/_2/VerbType.java +++ b/src/main/java/org/openarchives/oai/_2/VerbType.java @@ -8,9 +8,9 @@ package org.openarchives.oai._2; -import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlEnumValue; -import javax.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlEnum; +import jakarta.xml.bind.annotation.XmlEnumValue; +import jakarta.xml.bind.annotation.XmlType; /** diff --git a/src/main/java/org/openarchives/oai/_2/package-info.java b/src/main/java/org/openarchives/oai/_2/package-info.java index 1c432778..dbdf2eb3 100644 --- a/src/main/java/org/openarchives/oai/_2/package-info.java +++ b/src/main/java/org/openarchives/oai/_2/package-info.java @@ -4,9 +4,9 @@ // Änderungen an dieser Datei gehen bei einer Neukompilierung des Quellschemas verloren. // Generiert: 2016.08.23 um 12:48:44 PM CEST // -@javax.xml.bind.annotation.XmlSchema(xmlns = { - @javax.xml.bind.annotation.XmlNs(prefix = "xsi", namespaceURI = "http://www.w3.org/2001/XMLSchema-instance") +@jakarta.xml.bind.annotation.XmlSchema(xmlns = { + @jakarta.xml.bind.annotation.XmlNs(prefix = "xsi", namespaceURI = "http://www.w3.org/2001/XMLSchema-instance") }, namespace = "http://www.openarchives.org/OAI/2.0/", location = "http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd", - elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED) + elementFormDefault = jakarta.xml.bind.annotation.XmlNsForm.QUALIFIED) package org.openarchives.oai._2; diff --git a/src/main/resources/static/editor/lib/js/metadataeditor.js b/src/main/resources/static/editor/lib/js/metadataeditor.js index 8083c05e..abcac8fb 100644 --- a/src/main/resources/static/editor/lib/js/metadataeditor.js +++ b/src/main/resources/static/editor/lib/js/metadataeditor.js @@ -89,7 +89,6 @@ var buttons = { function generateIcon(fontAwesome, tooltip) { return "\n\"; } -; /** * Default table layout @@ -144,7 +143,7 @@ function modalTemplate(modalInput) { '' + ''; } -; + /** * describes the internal representation of the Editor in case a form will be generated. * @returns {editorDefinitionForm} diff --git a/src/main/resources/static/landingpage/js/download.js b/src/main/resources/static/landingpage/js/download.js new file mode 100644 index 00000000..934edccb --- /dev/null +++ b/src/main/resources/static/landingpage/js/download.js @@ -0,0 +1,162 @@ +//download.js v4.2, by dandavis; 2008-2016. [CCBY2] see http://danml.com/download.html for tests/usage +// v1 landed a FF+Chrome compat way of downloading strings to local un-named files, upgraded to use a hidden frame and optional mime +// v2 added named files via a[download], msSaveBlob, IE (10+) support, and window.URL support for larger+faster saves than dataURLs +// v3 added dataURL and Blob Input, bind-toggle arity, and legacy dataURL fallback was improved with force-download mime and base64 support. 3.1 improved safari handling. +// v4 adds AMD/UMD, commonJS, and plain browser support +// v4.1 adds url download capability via solo URL argument (same domain/CORS only) +// v4.2 adds semantic variable names, long (over 2MB) dataURL support, and hidden by default temp anchors +// https://github.com/rndme/download + +(function (root, factory) { + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define([], factory); + } else if (typeof exports === 'object') { + // Node. Does not work with strict CommonJS, but + // only CommonJS-like environments that support module.exports, + // like Node. + module.exports = factory(); + } else { + // Browser globals (root is window) + root.download = factory(); + } +}(this, function () { + + return function download(data, strFileName, strMimeType) { + + var self = window, // this script is only for browsers anyway... + defaultMime = "application/octet-stream", // this default mime also triggers iframe downloads + mimeType = strMimeType || defaultMime, + payload = data, + url = !strFileName && !strMimeType && payload, + anchor = document.createElement("a"), + toString = function(a){return String(a);}, + myBlob = (self.Blob || self.MozBlob || self.WebKitBlob || toString), + fileName = strFileName || "download", + blob, + reader; + myBlob= myBlob.call ? myBlob.bind(self) : Blob ; + + if(String(this)==="true"){ //reverse arguments, allowing download.bind(true, "text/xml", "export.xml") to act as a callback + payload=[payload, mimeType]; + mimeType=payload[0]; + payload=payload[1]; + } + + + if(url && url.length< 2048){ // if no filename and no mime, assume a url was passed as the only argument + fileName = url.split("/").pop().split("?")[0]; + anchor.href = url; // assign href prop to temp anchor + if(anchor.href.indexOf(url) !== -1){ // if the browser determines that it's a potentially valid url path: + var ajax=new XMLHttpRequest(); + ajax.open( "GET", url, true); + ajax.responseType = 'blob'; + ajax.onload= function(e){ + download(e.target.response, fileName, defaultMime); + }; + setTimeout(function(){ ajax.send();}, 0); // allows setting custom ajax headers using the return: + return ajax; + } // end if valid url? + } // end if url? + + + //go ahead and download dataURLs right away + if(/^data\:[\w+\-]+\/[\w+\-]+[,;]/.test(payload)){ + + if(payload.length > (1024*1024*1.999) && myBlob !== toString ){ + payload=dataUrlToBlob(payload); + mimeType=payload.type || defaultMime; + }else{ + return navigator.msSaveBlob ? // IE10 can't do a[download], only Blobs: + navigator.msSaveBlob(dataUrlToBlob(payload), fileName) : + saver(payload) ; // everyone else can save dataURLs un-processed + } + + }//end if dataURL passed? + + blob = payload instanceof myBlob ? + payload : + new myBlob([payload], {type: mimeType}) ; + + + function dataUrlToBlob(strUrl) { + var parts= strUrl.split(/[:;,]/), + type= parts[1], + decoder= parts[2] == "base64" ? atob : decodeURIComponent, + binData= decoder( parts.pop() ), + mx= binData.length, + i= 0, + uiArr= new Uint8Array(mx); + + for(i;i - + @@ -10,11 +10,11 @@ - - - - - + + + + + @@ -40,7 +40,7 @@

Schema Management

- List of the schema records + List of the schema records @@ -50,7 +50,7 @@

Schema Management

Metadata Management

- List of the metadata records + List of the metadata records @@ -72,7 +72,7 @@

MetaRepo GUI Documentation

- - + + diff --git a/src/main/resources/templates/fragments/_navbar.html b/src/main/resources/templates/fragments/_navbar.html index 74310f37..0ef83aa6 100644 --- a/src/main/resources/templates/fragments/_navbar.html +++ b/src/main/resources/templates/fragments/_navbar.html @@ -1,13 +1,15 @@ - -
+ + + + diff --git a/src/main/resources/templates/metadata-landing-page.html b/src/main/resources/templates/metadata-landing-page.html new file mode 100644 index 00000000..ac0df88e --- /dev/null +++ b/src/main/resources/templates/metadata-landing-page.html @@ -0,0 +1,144 @@ + + + + + + Landingpage for Metadata document + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+ + +
+
+ Landingpage for Metadata Document(s) +
+
+ +
+ +
+
+

Entries

+
+
+ + + + + + + + + + + + + +
ID VersionTypeLast Update Related ResourceMetadataSchema
............ + + + +
+
+
+
+
+ + diff --git a/src/main/resources/templates/metadata-management.html b/src/main/resources/templates/metadata-management.html index 52290166..8f53d4b1 100644 --- a/src/main/resources/templates/metadata-management.html +++ b/src/main/resources/templates/metadata-management.html @@ -1,21 +1,21 @@ - + Metadata Management - - - - - - + + + + + + @@ -32,7 +32,7 @@ Metadata Management
@@ -71,15 +71,15 @@

Metadata Record Form

- - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+ + +
+
+ Landingpage for Schema Document(s) +
+
+ +
+ +
+
+

Entries

+
+
+ + + + + + + + + + + + + + +
ID VersionTypeLast Update LabelDefinitionCommentSchema
..................... + +
+
+
+
+
+ + diff --git a/src/main/resources/templates/schema-management.html b/src/main/resources/templates/schema-management.html index 2d5c33d2..5a0453ea 100644 --- a/src/main/resources/templates/schema-management.html +++ b/src/main/resources/templates/schema-management.html @@ -1,20 +1,20 @@ - + Schema Management - - - - - - + + + + + + @@ -31,7 +31,7 @@ Schema Management
    -
  2. Schema Management
@@ -70,15 +70,15 @@

Schema Record Form

- - - - - - - - - + + + + + + + + +