Optmize prefetch for mass ingestion transactions (#6589)

* Optmize prefetch for mass ingestion transactions * Add changelog * Spotless * Resolve review comments and spotless
hapifhir · Jan 7, 2025 · 12dd393 · 12dd393
1 parent d6b584b
commit 12dd393
Show file tree

Hide file tree

Showing 6 changed files with 182 additions and 104 deletions.
diff --git a/...ources/ca/uhn/hapi/fhir/changelog/7_8_0/6589_optimize_transactions_in_mass_ingestion_mode b/...ources/ca/uhn/hapi/fhir/changelog/7_8_0/6589_optimize_transactions_in_mass_ingestion_mode
@@ -0,0 +1,4 @@
+---
+type: perf
+issue: 6589
+title: "When performing data loading into a JPA repository using FHIR transactions with Mass Ingestion Mode enabled, the prefetch routine has been optimized to avoid loading the current resource body/contents, since these are not actually needed in Mass Ingestion mode. This avoids a redundant select statement being issued for each transaction and should improve performance."
diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirSystemDao.java
@@ -206,12 +206,42 @@ public <P extends IResourcePersistentId> void preFetchResources(
 			 * However, for realistic average workloads, this should reduce the number of round trips.
 			 */
 			if (!idChunk.isEmpty()) {
-				List<ResourceTable> entityChunk = prefetchResourceTableAndHistory(idChunk);
+				List<ResourceTable> entityChunk = null;
+
+				/*
+				 * Unless we're in Mass Ingestion mode, we will pre-fetch the current
+				 * saved resource text in HFJ_RES_VER (ResourceHistoryTable). If we're
+				 * in Mass Ingestion Mode, we don't need to do that because every update
+				 * will generate a new version anyway so the system never needs to know
+				 * the current contents.
+				 */
+				if (!myStorageSettings.isMassIngestionMode()) {
+					entityChunk = prefetchResourceTableAndHistory(idChunk);
+				}
 
 				if (thePreFetchIndexes) {
 
+					/*
+					 * If we're in mass ingestion mode, then we still need to load the resource
+					 * entries in HFJ_RESOURCE (ResourceTable). We combine that with the search
+					 * for tokens (since token is the most likely kind of index to be populated
+					 * for any arbitrary resource type).
+					 *
+					 * For all other index types, we only load indexes if at least one
+					 * HFJ_RESOURCE row indicates that a resource we care about actually has
+					 * index rows of the given type.
+					 */
+					if (entityChunk == null) {
+						String jqlQuery =
+								"SELECT r FROM ResourceTable r LEFT JOIN FETCH r.myParamsToken WHERE r.myPid IN ( :IDS )";
+						TypedQuery<ResourceTable> query = myEntityManager.createQuery(jqlQuery, ResourceTable.class);
+						query.setParameter("IDS", idChunk);
+						entityChunk = query.getResultList();
+					} else {
+						prefetchByField("token", "myParamsToken", ResourceTable::isParamsTokenPopulated, entityChunk);
+					}
+
 					prefetchByField("string", "myParamsString", ResourceTable::isParamsStringPopulated, entityChunk);
-					prefetchByField("token", "myParamsToken", ResourceTable::isParamsTokenPopulated, entityChunk);
 					prefetchByField("date", "myParamsDate", ResourceTable::isParamsDatePopulated, entityChunk);
 					prefetchByField(
 							"quantity", "myParamsQuantity", ResourceTable::isParamsQuantityPopulated, entityChunk);

diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/TransactionProcessor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/TransactionProcessor.java
@@ -173,7 +173,7 @@ protected EntriesToProcessMap doTransactionWriteOperations(
 		 * is for fast writing of data.
 		 *
 		 * Note that it's probably not necessary to reset it back, it should
-		 * automatically go back to the default value after the transaction but
+		 * automatically go back to the default value after the transaction, but
 		 * we reset it just to be safe.
 		 */
 		FlushModeType initialFlushMode = myEntityManager.getFlushMode();