merge master

.github/workflows/cdr_check/pyproject.toml

@@ -3,6 +3,7 @@ name = "tester"
 version = "0.1.0"
 description = ""
 authors = ["Tadgh <garygrantgraham@gmail.com>"]
+package-mode = false
 
 [tool.poetry.dependencies]
 python = "^3.10"

hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java

@@ -137,6 +137,15 @@ public class FhirContext {
 	private volatile Boolean myFormatNDJsonSupported;
 	private volatile Boolean myFormatRdfSupported;
 	private IFhirValidatorFactory myFhirValidatorFactory = FhirValidator::new;
+	/**
+	 * If true, parsed resources will have the json string
+	 * used to create them stored
+	 * in the UserData.
+	 *
+	 * This is to help with validation, because the parser itself is far
+	 * more lenient than validation might be.
+	 */
+	private boolean myStoreResourceJsonFlag = false;
 
 	/**
 	 * @deprecated It is recommended that you use one of the static initializer methods instead
@@ -762,6 +771,14 @@ public void setValidationSupport(IValidationSupport theValidationSupport) {
 		myValidationSupport = theValidationSupport;
 	}
 
+	public void setStoreRawJson(boolean theStoreResourceJsonFlag) {
+		myStoreResourceJsonFlag = theStoreResourceJsonFlag;
+	}
+
+	public boolean isStoreResourceJson() {
+		return myStoreResourceJsonFlag;
+	}
+
 	public IFhirVersion getVersion() {
 		return myVersion;
 	}

hapi-fhir-base/src/main/java/ca/uhn/fhir/narrative/CustomThymeleafNarrativeGenerator.java

@@ -66,6 +66,10 @@ public CustomThymeleafNarrativeGenerator(List<String> theNarrativePropertyFiles)
 		this(theNarrativePropertyFiles.toArray(new String[0]));
 	}
 
+	public CustomThymeleafNarrativeGenerator(CustomThymeleafNarrativeGenerator theNarrativeGenerator) {
+		setManifest(theNarrativeGenerator.getManifest());
+	}
+
 	@Override
 	public NarrativeTemplateManifest getManifest() {
 		NarrativeTemplateManifest retVal = myManifest;

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/BaseParser.java

@@ -46,6 +46,7 @@
 import ca.uhn.fhir.util.CollectionUtil;
 import ca.uhn.fhir.util.FhirTerser;
 import ca.uhn.fhir.util.MetaUtil;
+import ca.uhn.fhir.util.ResourceUtil;
 import ca.uhn.fhir.util.UrlUtil;
 import com.google.common.base.Charsets;
 import jakarta.annotation.Nullable;
@@ -640,12 +641,30 @@ public <T extends IBaseResource> T parseResource(Class<T> theResourceType, Reade
 		 * We do this so that the context can verify that the structure is for
 		 * the correct FHIR version
 		 */
+		Reader readerToUse = theReader;
 		if (theResourceType != null) {
 			myContext.getResourceDefinition(theResourceType);
+			if (myContext.isStoreResourceJson()) {
+				readerToUse = new PreserveStringReader(theReader);
+			}
 		}
 
 		// Actually do the parse
-		T retVal = doParseResource(theResourceType, theReader);
+		T retVal = doParseResource(theResourceType, readerToUse);
+
+		if (theResourceType != null && myContext.isStoreResourceJson()) {
+			PreserveStringReader psr = (PreserveStringReader) readerToUse;
+			if (psr.hasString()) {
+				try {
+					ResourceUtil.addRawDataToResource(retVal, getEncoding(), psr.toString());
+					psr.close();
+				} catch (IOException ex) {
+					ourLog.warn(
+							"Unable to store raw JSON on resource. This won't affect functionality, but validation will use the resource itself, which may result in different validation results than a $validation operation.",
+							ex);
+				}
+			}
+		}
 
 		RuntimeResourceDefinition def = myContext.getResourceDefinition(retVal);
 		if ("Bundle".equals(def.getName())) {

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/PreserveStringReader.java

@@ -0,0 +1,44 @@
+package ca.uhn.fhir.parser;
+
+import jakarta.annotation.Nonnull;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringWriter;
+
+public class PreserveStringReader extends Reader {
+
+	private final Reader myReader;
+
+	private final StringWriter myWriter;
+
+	public PreserveStringReader(Reader theReader) {
+		super(theReader);
+		myReader = theReader;
+		myWriter = new StringWriter();
+	}
+
+	@Override
+	public int read(@Nonnull char[] theBuffer, int theOffset, int theLength) throws IOException {
+		int out = myReader.read(theBuffer, theOffset, theLength);
+		if (out >= 0) {
+			myWriter.write(theBuffer, theOffset, out);
+		}
+
+		return out;
+	}
+
+	@Override
+	public void close() throws IOException {
+		myReader.close();
+		myWriter.close();
+	}
+
+	public boolean hasString() {
+		return myWriter.getBuffer().length() > 0;
+	}
+
+	public String toString() {
+		return myWriter.toString();
+	}
+}

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ResourceUtil.java

@@ -22,11 +22,20 @@
 import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
 import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
 import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.rest.api.EncodingEnum;
+import jakarta.annotation.Nonnull;
 import org.hl7.fhir.instance.model.api.IBaseBundle;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 
+import java.io.IOException;
+
 public class ResourceUtil {
 
+	private static final String ENCODING = "ENCODING_TYPE";
+	private static final String RAW_ = "RAW_%s";
+
+	private ResourceUtil() {}
+
 	/**
 	 * This method removes the narrative from the resource, or if the resource is a bundle, removes the narrative from
 	 * all of the resources in the bundle
@@ -47,4 +56,27 @@ public static void removeNarrative(FhirContext theContext, IBaseResource theInpu
 			textElement.getMutator().setValue(theInput, null);
 		}
 	}
+
+	public static void addRawDataToResource(
+			@Nonnull IBaseResource theResource, @Nonnull EncodingEnum theEncodingType, String theSerializedData)
+			throws IOException {
+		theResource.setUserData(getRawUserDataKey(theEncodingType), theSerializedData);
+		theResource.setUserData(ENCODING, theEncodingType);
+	}
+
+	public static EncodingEnum getEncodingTypeFromUserData(@Nonnull IBaseResource theResource) {
+		return (EncodingEnum) theResource.getUserData(ENCODING);
+	}
+
+	public static String getRawStringFromResourceOrNull(@Nonnull IBaseResource theResource) {
+		EncodingEnum type = (EncodingEnum) theResource.getUserData(ENCODING);
+		if (type != null) {
+			return (String) theResource.getUserData(getRawUserDataKey(type));
+		}
+		return null;
+	}
+
+	private static String getRawUserDataKey(EncodingEnum theEncodingEnum) {
+		return String.format(RAW_, theEncodingEnum.name());
+	}
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/validation/ValidationContext.java

@@ -26,7 +26,9 @@
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.util.ObjectUtil;
+import ca.uhn.fhir.util.ResourceUtil;
 import jakarta.annotation.Nonnull;
+import org.apache.commons.lang3.ObjectUtils;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 
 import java.util.ArrayList;
@@ -103,12 +105,17 @@ public static <T extends IBaseResource> IValidationContext<T> forResource(
 		IEncoder encoder = new IEncoder() {
 			@Override
 			public String encode() {
-				return theContext.newJsonParser().encodeResourceToString(theResource);
+				// use the stored json string, if available
+				// otherwise, encode the actual resource
+				return ObjectUtils.firstNonNull(
+						ResourceUtil.getRawStringFromResourceOrNull(theResource),
+						theContext.newJsonParser().encodeResourceToString(theResource));
 			}
 
 			@Override
 			public EncodingEnum getEncoding() {
-				return EncodingEnum.JSON;
+				return ObjectUtils.defaultIfNull(
+						ResourceUtil.getEncodingTypeFromUserData(theResource), EncodingEnum.JSON);
 			}
 		};
 		return new ValidationContext<>(theContext, theResource, encoder, options);

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/version.yaml

@@ -1,3 +1,3 @@
 ---
 release-date: "2024-11-15"
-codename: "TBD"
+codename: "Despina"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_1/6424-change-structure-definitions-to-never-expire-in-cache-during-validation.yaml

@@ -0,0 +1,5 @@
+---
+type: change
+issue: 6424
+title: "Changed VersionSpecificWorkerContextWrapper to never expire StructureDefinition entries in the cache, 
+which is needed because the validator makes assumptions about StructureDefinitions never changing."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_1/6538-improve-transaction-performance.yaml

@@ -0,0 +1,7 @@
+---
+type: perf
+issue: 6538
+title: "In HAPI FHIR 8.0.0, transaction processing has been significantly improved thanks
+   to ticket [#6460](https://github.com/hapifhir/hapi-fhir/pull/6460). This enhancement
+   has been partially backported to the 7.6.x release line in order to provide partial improvement
+   prior to the release of HAPI FHIR 8.0.0."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_1/6538-improve-validation-performance.yaml

@@ -0,0 +1,7 @@
+---
+type: perf
+issue: 6538
+title: "In HAPI FHIR 8.0.0, validation processing has been significantly improved thanks
+   to ticket [#6508](https://github.com/hapifhir/hapi-fhir/pull/6508). This enhancement
+   has been partially backported to the 7.6.x release line in order to provide partial improvement
+   prior to the release of HAPI FHIR 8.0.0."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_1/upgrade.md

@@ -0,0 +1,12 @@
+## Device membership in Patient Compartment
+
+As of 7.6.1, versions of FHIR below R5 now consider the `Device` resource's `patient` Search Parameter to be in the Patient Compartment. The following features are affected:
+
+- Patient Search with `_revInclude=*`
+- Patient instance-level `$everything` operation
+- Patient type-level `$everything` operation
+- Automatic Search Narrowing
+- Bulk Export
+
+Previously, there were various shims in the code that permitted similar behaviour in these features. Those shims have been removed. The only remaining component is [Advanced Compartment Authorization](/hapi-fhir/docs/security/authorization_interceptor.html#advanced-compartment-authorization), which can still be used 
+to add other Search Parameters into a given compartment.

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_1/version.yaml

@@ -0,0 +1,3 @@
+---
+release-date: "2024-12-18"
+codename: "Despina"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6536-make-device-part-of-patient-compartment.yaml

@@ -1,6 +1,7 @@
 ---
 type: add
 jira: SMILE-9260
+backport: 7.6.1
 title: "The `patient` search parameter for the `Device` resource has been added to the Patient Compartment for the purposes of: 
 
 - AuthorizationInterceptor

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6574-storing-raw-json-on-resource-parsing.yaml

@@ -0,0 +1,10 @@
+---
+type: fix
+issue: 6574
+jira: SMILE-9432
+title: "The raw json of parsed resources will be kept in the UserData
+        (key: `RAW_JSON`) of the resource itself.
+        This is to allow consistency in handling validation downstream,
+        since otherwise the FhirParser is far more lenient about what
+        it can parse than $validate is for what it accepts.
+"

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6580-migration-result-column.yaml

@@ -0,0 +1,7 @@
+---
+type: add
+issue: 6580
+title: "A new `RESULT` column has been added to the database migration table to record the migration execution result.
+values are `NOT_APPLIED_SKIPPED` (either skipped via the `skip-versions` flag or if the migration task was stubbed), 
+`NOT_APPLIED_NOT_FOR_THIS_DATABASE` (does not apply to that database), `NOT_APPLIED_PRECONDITION_NOT_MET` (not run based on a SQL script outcome),
+`NOT_APPLIED_ALLOWED_FAILURE` (the migration failed, but it is permitted to fail), `APPLIED`."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6587-enhance-vital-signs-narrative-generation-template.yaml

@@ -0,0 +1,5 @@
+---
+type: add
+issue: 6587
+title: "Enhanced the IPS vital signs narrative template to include code and value information for 
+    all entries in the `Observation.component` property."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_8_0/6589_optimize_transactions_in_mass_ingestion_mode

@@ -0,0 +1,4 @@
+---
+type: perf
+issue: 6589
+title: "When performing data loading into a JPA repository using FHIR transactions with Mass Ingestion Mode enabled, the prefetch routine has been optimized to avoid loading the current resource body/contents, since these are not actually needed in Mass Ingestion mode. This avoids a redundant select statement being issued for each transaction and should improve performance."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/ValidationSupportConfig.java

@@ -25,6 +25,7 @@
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.dao.JpaPersistedResourceValidationSupport;
+import ca.uhn.fhir.jpa.validation.FhirContextValidationSupportSvc;
 import ca.uhn.fhir.jpa.validation.ValidatorPolicyAdvisor;
 import ca.uhn.fhir.jpa.validation.ValidatorResourceFetcher;
 import ca.uhn.fhir.validation.IInstanceValidatorModule;
@@ -42,6 +43,11 @@ public class ValidationSupportConfig {
 	@Autowired
 	private FhirContext myFhirContext;
 
+	@Bean
+	public FhirContextValidationSupportSvc fhirValidationSupportSvc() {
+		return new FhirContextValidationSupportSvc(myFhirContext);
+	}
+
 	@Bean(name = JpaConfig.DEFAULT_PROFILE_VALIDATION_SUPPORT)
 	public DefaultProfileValidationSupport defaultProfileValidationSupport() {
 		return new DefaultProfileValidationSupport(myFhirContext);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java

@@ -206,12 +206,42 @@ public <P extends IResourcePersistentId> void preFetchResources(
 			 * However, for realistic average workloads, this should reduce the number of round trips.
 			 */
 			if (!idChunk.isEmpty()) {
-				List<ResourceTable> entityChunk = prefetchResourceTableAndHistory(idChunk);
+				List<ResourceTable> entityChunk = null;
+
+				/*
+				 * Unless we're in Mass Ingestion mode, we will pre-fetch the current
+				 * saved resource text in HFJ_RES_VER (ResourceHistoryTable). If we're
+				 * in Mass Ingestion Mode, we don't need to do that because every update
+				 * will generate a new version anyway so the system never needs to know
+				 * the current contents.
+				 */
+				if (!myStorageSettings.isMassIngestionMode()) {
+					entityChunk = prefetchResourceTableAndHistory(idChunk);
+				}
 
 				if (thePreFetchIndexes) {
 
+					/*
+					 * If we're in mass ingestion mode, then we still need to load the resource
+					 * entries in HFJ_RESOURCE (ResourceTable). We combine that with the search
+					 * for tokens (since token is the most likely kind of index to be populated
+					 * for any arbitrary resource type).
+					 *
+					 * For all other index types, we only load indexes if at least one
+					 * HFJ_RESOURCE row indicates that a resource we care about actually has
+					 * index rows of the given type.
+					 */
+					if (entityChunk == null) {
+						String jqlQuery =
+								"SELECT r FROM ResourceTable r LEFT JOIN FETCH r.myParamsToken WHERE r.myPid IN ( :IDS )";
+						TypedQuery<ResourceTable> query = myEntityManager.createQuery(jqlQuery, ResourceTable.class);
+						query.setParameter("IDS", idChunk);
+						entityChunk = query.getResultList();
+					} else {
+						prefetchByField("token", "myParamsToken", ResourceTable::isParamsTokenPopulated, entityChunk);
+					}
+
 					prefetchByField("string", "myParamsString", ResourceTable::isParamsStringPopulated, entityChunk);
-					prefetchByField("token", "myParamsToken", ResourceTable::isParamsTokenPopulated, entityChunk);
 					prefetchByField("date", "myParamsDate", ResourceTable::isParamsDatePopulated, entityChunk);
 					prefetchByField(
 							"quantity", "myParamsQuantity", ResourceTable::isParamsQuantityPopulated, entityChunk);

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/TransactionProcessor.java

@@ -173,7 +173,7 @@ protected EntriesToProcessMap doTransactionWriteOperations(
 		 * is for fast writing of data.
 		 *
 		 * Note that it's probably not necessary to reset it back, it should
-		 * automatically go back to the default value after the transaction but
+		 * automatically go back to the default value after the transaction, but
 		 * we reset it just to be safe.
 		 */
 		FlushModeType initialFlushMode = myEntityManager.getFlushMode();

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/validation/FhirContextValidationSupportSvc.java

@@ -0,0 +1,19 @@
+package ca.uhn.fhir.jpa.validation;
+
+import ca.uhn.fhir.context.FhirContext;
+
+/**
+ * This bean will set our context to store the raw json of parsed
+ * resources when they come in for creation.
+ * -
+ * This is done so we can ensure that validation is done correctly,
+ * and the JSONParser is far more lenient than our validators.
+ * -
+ * See {@link FhirContext#isStoreResourceJson()}
+ */
+public class FhirContextValidationSupportSvc {
+
+	public FhirContextValidationSupportSvc(FhirContext theContext) {
+		theContext.setStoreRawJson(true);
+	}
+}