Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
Pfeil committed Nov 19, 2024
1 parent c7e6cbb commit 6b07c6f
Show file tree
Hide file tree
Showing 13 changed files with 386 additions and 34 deletions.
2 changes: 1 addition & 1 deletion src/main/java/edu/kit/datamanager/pit/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public class Application {
protected static final String ERROR_COMMUNICATION = "Communication error: {}";
protected static final String ERROR_CONFIGURATION = "Configuration error: {}";

protected static final Executor EXECUTOR = Executors.newWorkStealingPool();
public static final Executor EXECUTOR = Executors.newVirtualThreadPerTaskExecutor();


@Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ public RecordValidationException(PIDRecord pidRecord, String reason) {
this.pidRecord = pidRecord;
}

public RecordValidationException(PIDRecord pidRecord, String reason, Exception e) {
super(HTTP_STATUS, VALIDATION_OF_RECORD + pidRecord.getPid() + " failed. Reason:\n" + reason, e);
this.pidRecord = pidRecord;
}

public PIDRecord getPidRecord() {
return pidRecord;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
import edu.kit.datamanager.pit.pitservice.impl.NoValidationStrategy;

import java.net.URL;
import java.util.List;
import java.util.Set;

import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import jakarta.validation.constraints.NotNull;

import org.springframework.beans.factory.annotation.Value;
Expand All @@ -46,6 +49,11 @@
@Validated
public class ApplicationProperties extends GenericApplicationProperties {

private static Set<String> KNOWN_PROFILE_KEYS = Set.of(
"21.T11148/076759916209e5d62bd5",
"21.T11969/bcc54a2a9ab5bf2a8f2c"
);

public enum IdentifierSystemImpl {
IN_MEMORY,
LOCAL,
Expand All @@ -66,10 +74,10 @@ public enum ValidationStrategy {
private ValidationStrategy validationStrategy = ValidationStrategy.EMBEDDED_STRICT;

@Bean
public IValidationStrategy defaultValidationStrategy() {
public IValidationStrategy defaultValidationStrategy(ITypeRegistry typeRegistry) {
IValidationStrategy defaultStrategy = new NoValidationStrategy();
if (this.validationStrategy == ValidationStrategy.EMBEDDED_STRICT) {
defaultStrategy = new EmbeddedStrictValidatorStrategy();
defaultStrategy = new EmbeddedStrictValidatorStrategy(typeRegistry, this);
}
return defaultStrategy;
}
Expand Down Expand Up @@ -108,8 +116,24 @@ public boolean storesResolved() {
private long expireAfterWrite;

@Value("${pit.validation.profileKey:21.T11148/076759916209e5d62bd5}")
@Deprecated(forRemoval = true /*In Typed PID Maker 3.0.0*/)
private String profileKey;

public @NotNull Set<String> getProfileKeys() {
Set<String> allProfileKeys = new java.util.HashSet<>(Set.copyOf(KNOWN_PROFILE_KEYS));
allProfileKeys.addAll(profileKeys);
allProfileKeys.add(this.getProfileKey());
return allProfileKeys;
}

public void setProfileKeys(@NotNull List<String> profileKeys) {
this.profileKeys = profileKeys;
}

@Value("#{${pit.validation.profileKeys:{}}}")
@NotNull
protected List<String> profileKeys = List.of();

public IdentifierSystemImpl getIdentifierSystemImplementation() {
return this.identifierSystemImplementation;
}
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/edu/kit/datamanager/pit/domain/ImmutableList.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package edu.kit.datamanager.pit.domain;

import java.util.Collections;
import java.util.List;

public record ImmutableList<I>(List<I> items) {
public ImmutableList {
items = Collections.unmodifiableList(items);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ public void removeAllValuesOf(String attribute) {
* @param profile the given Profile definition.
* @return all missing mandatory attributes.
*/
@Deprecated
public Collection<String> getMissingMandatoryTypesOf(TypeDefinition profile) {
Collection<String> missing = new ArrayList<>();
for (TypeDefinition td : profile.getSubTypes().values()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,26 @@
package edu.kit.datamanager.pit.pitservice.impl;

import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import edu.kit.datamanager.pit.common.ExternalServiceException;
import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.configuration.ApplicationProperties;
import edu.kit.datamanager.pit.domain.PIDRecord;
import edu.kit.datamanager.pit.domain.TypeDefinition;
import edu.kit.datamanager.pit.pitservice.IValidationStrategy;
import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
import edu.kit.datamanager.pit.typeregistry.RegisteredProfile;
import edu.kit.datamanager.pit.util.TypeValidationUtils;

import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.*;
import java.util.stream.Collectors;

import org.apache.commons.lang3.stream.Streams;
import org.everit.json.schema.ValidationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

/**
* Validates a PID record using embedded profile(s).
Expand All @@ -31,33 +34,71 @@ public class EmbeddedStrictValidatorStrategy implements IValidationStrategy {
private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class);
protected static final Executor EXECUTOR = Executors.newWorkStealingPool();

@Autowired
public AsyncLoadingCache<String, TypeDefinition> typeLoader;
protected ITypeRegistry typeRegistry;
// TODO store extracted information only
protected ApplicationProperties config;
protected boolean additionalAttributesAllowed;
protected Set<String> profileKeys;

@Autowired
ApplicationProperties applicationProps;
public EmbeddedStrictValidatorStrategy(ITypeRegistry typeRegistry, ApplicationProperties config) {
this.typeRegistry = typeRegistry;
this.config = config;
}

@Override
public void validate(PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException {
String profileKey = applicationProps.getProfileKey();
if (!pidRecord.hasProperty(profileKey)) {

if (pidRecord.getPropertyIdentifiers().stream().anyMatch(this.profileKeys::contains)) {
// TODO to support records without referenced profiles, we could consider skipping this...
throw new RecordValidationException(
pidRecord,
"Profile attribute not found. Expected key: " + profileKey);
"Profile attribute not found. Expected key: " + this.profileKeys);
}

String[] profilePIDs = pidRecord.getPropertyValues(profileKey);
// For each attribute in record, resolve schema and check the value
List<CompletableFuture<AttributeInfo>> attributeInfoFutures = pidRecord.getPropertyIdentifiers().stream()
.map(attributePid -> this.typeRegistry.queryAttributeInfo(attributePid))
.map(attributeInfoFuture -> attributeInfoFuture.thenApply(attributeInfo -> {
for (String value : pidRecord.getPropertyValues(attributeInfo.pid())) {
try {
attributeInfo.jsonSchema().validate(value);
} catch (ValidationException e) {
throw new RecordValidationException(
pidRecord,
"Attribute %s has a non-complying value %s".formatted(attributeInfo.pid(), value),
e
);
}
}
return attributeInfo;
}))
.map(attributeInfoFuture -> attributeInfoFuture.thenApply(attributeInfo -> {
boolean isProfile = this.profileKeys.contains(attributeInfo.pid());
if (isProfile) {
Arrays.stream(pidRecord.getPropertyValues(attributeInfo.pid()))
.map(profilePid -> this.typeRegistry.queryAsProfile(profilePid))
.forEach(registeredProfileFuture -> registeredProfileFuture.thenApply(registeredProfile -> {
registeredProfile.validateAttributes(pidRecord, this.additionalAttributesAllowed);
return registeredProfile;
}));
}
}))
.toList();

CompletableFuture.allOf(attributeInfoFutures.toArray(new CompletableFuture<?>[0])).join();

String[] profilePIDs = pidRecord.getPropertyValues(this.profileKeys);
boolean hasProfile = profilePIDs.length > 0;
if (!hasProfile) {
throw new RecordValidationException(
pidRecord,
"Profile attribute " + profileKey + " has no values.");
"Profile attribute " + this.profileKeys + " has no values.");
}

List<CompletableFuture<?>> futures = Streams.stream(Arrays.stream(profilePIDs))
List<CompletableFuture<?>> futures = Streams.failableStream(Arrays.stream(profilePIDs))
.map(profilePID -> {
try {
return this.typeLoader.get(profilePID)
return this.typeRegistry.queryAsProfile(profilePID)
.thenAcceptAsync(profileDefinition -> {
if (profileDefinition == null) {
LOG.error("No type definition found for identifier {}.", profilePID);
Expand All @@ -70,7 +111,7 @@ public void validate(PIDRecord pidRecord) throws RecordValidationException, Exte
} catch (RuntimeException e) {
LOG.error("Could not resolve identifier {}.", profilePID);
throw new ExternalServiceException(
applicationProps.getTypeRegistryUri().toString());
this.typeRegistry.getRegistryUrl().toString());
}
})
.collect(Collectors.toList());
Expand All @@ -79,7 +120,7 @@ public void validate(PIDRecord pidRecord) throws RecordValidationException, Exte
} catch (CompletionException e) {
throwRecordValidationExceptionCause(e);
throw new ExternalServiceException(
applicationProps.getTypeRegistryUri().toString());
this.typeRegistry.getRegistryUrl().toString());
} catch (CancellationException e) {
throwRecordValidationExceptionCause(e);
throw new RecordValidationException(
Expand All @@ -101,22 +142,22 @@ private static void throwRecordValidationExceptionCause(Throwable e) {
* @param profile the profile to validate against.
* @throws RecordValidationException with error message on validation errors.
*/
private void strictProfileValidation(PIDRecord pidRecord, TypeDefinition profile) throws RecordValidationException {
private void strictProfileValidation(PIDRecord pidRecord, RegisteredProfile profile) throws RecordValidationException {
// if (profile.hasSchema()) {
// TODO issue https://github.com/kit-data-manager/pit-service/issues/104
// validate using schema and you are done (strict validation)
// String jsonRecord = ""; // TODO format depends on schema source
// return profile.validate(jsonRecord);
// }

LOG.trace("Validating PID record against profile {}.", profile.getIdentifier());
LOG.trace("Validating PID record against profile {}.", profile.pid());

TypeValidationUtils.checkMandatoryAttributes(pidRecord, profile);

for (String attributeKey : pidRecord.getPropertyIdentifiers()) {
LOG.trace("Checking PID record key {}.", attributeKey);

TypeDefinition type = profile.getSubTypes().get(attributeKey);
TypeDefinition type = profile.attributes().items().get(attributeKey);
if (type == null) {
LOG.error("No sub-type found for key {}.", attributeKey);
// TODO try to resolve it (for later when we support "allow additional
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package edu.kit.datamanager.pit.typeregistry;

import org.everit.json.schema.Schema;

import java.util.List;

/**
* @param pid the pid of this attribute
* @param typeName name of the schema type of this attribute in the DTR,
* e.g. "Profile", "InfoType", "Special-Info-Type", ...
* @param jsonSchema the json schema to validate a value of this attribute
*/
public record AttributeInfo(
String pid,
String typeName,
Schema jsonSchema
) {}
Original file line number Diff line number Diff line change
@@ -1,24 +1,19 @@
package edu.kit.datamanager.pit.typeregistry;

import java.io.IOException;
import org.everit.json.schema.Schema;

import edu.kit.datamanager.pit.domain.TypeDefinition;

import java.net.URISyntaxException;
import java.net.URL;
import java.util.concurrent.CompletableFuture;

/**
* Main abstraction interface towards the type registry. Contains all methods
* required from the registry by the core services.
*
*/
public interface ITypeRegistry {

/**
* Queries a type definition record from the type registry.
*
* @param typeIdentifier
* @return a type definition record or null if the type is not registered.
* @throws IOException on communication errors with a remote registry
*/
public TypeDefinition queryTypeDefinition(String typeIdentifier) throws IOException, URISyntaxException;
@Deprecated
CompletableFuture<Schema> queryAttributeSchemaOf(String attributePid);
CompletableFuture<AttributeInfo> queryAttributeInfo(String attributePid);
CompletableFuture<RegisteredProfile> queryAsProfile(String profilePid);
URL getRegistryUrl();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package edu.kit.datamanager.pit.typeregistry;

import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.domain.ImmutableList;
import edu.kit.datamanager.pit.domain.PIDRecord;

import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

public record RegisteredProfile(
String pid,
ImmutableList<RegisteredProfileAttribute> attributes
) {
public void validateAttributes(PIDRecord pidRecord, boolean allowAdditionalAttributes) {
Set<String> additionalAttributes = pidRecord.getPropertyIdentifiers().stream()
.filter(recordKey -> attributes.items().stream().anyMatch(
profileAttribute -> Objects.equals(profileAttribute.pid(), recordKey)))
.collect(Collectors.toSet());

boolean violatesAdditionalAttributes = !allowAdditionalAttributes && !additionalAttributes.isEmpty();
if (violatesAdditionalAttributes) {
throw new RecordValidationException(
pidRecord,
String.format("Attributes %s are not allowed in profile %s",
String.join(", ", additionalAttributes),
this.pid)
);
}

for (RegisteredProfileAttribute profileAttribute : this.attributes.items()) {
if (profileAttribute.violatesMandatoryProperty(pidRecord)) {
throw new RecordValidationException(
pidRecord,
String.format("Attribute %s missing, but is mandatory in profile %s",
profileAttribute.pid(),
this.pid)
);
}
if (profileAttribute.violatesRepeatableProperty(pidRecord)) {
throw new RecordValidationException(
pidRecord,
String.format("Attribute %s is not repeatable in profile %s, but has multiple values",
profileAttribute.pid(),
this.pid)
);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package edu.kit.datamanager.pit.typeregistry;

import edu.kit.datamanager.pit.common.RecordValidationException;
import edu.kit.datamanager.pit.domain.PIDRecord;

public record RegisteredProfileAttribute(
String pid,
boolean mandatory,
boolean repeatable
) {
public boolean violatesMandatoryProperty(PIDRecord pidRecord) {
boolean contains = pidRecord.getPropertyIdentifiers().contains(this.pid)
&& pidRecord.getPropertyValues(this.pid).length > 0;
return this.mandatory && !contains;
}

public boolean violatesRepeatableProperty(PIDRecord pidRecord) {
boolean repeats = pidRecord.getPropertyValues(this.pid).length > 1;
return !this.repeatable && repeats;
}
}
Loading

0 comments on commit 6b07c6f

Please sign in to comment.