From 9116ff733113d6f8862916801f25343d505761e9 Mon Sep 17 00:00:00 2001 From: David Kinzer Date: Wed, 16 Oct 2024 12:56:34 -0400 Subject: [PATCH] Add missing data env to update/delete harvest task. This env variable is required in order to iterate over deleted and updated marc xml files. --- cob_datapipeline/catalog_production_oai_harvest_dag.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cob_datapipeline/catalog_production_oai_harvest_dag.py b/cob_datapipeline/catalog_production_oai_harvest_dag.py index d2b3f609..4484a5b2 100644 --- a/cob_datapipeline/catalog_production_oai_harvest_dag.py +++ b/cob_datapipeline/catalog_production_oai_harvest_dag.py @@ -189,6 +189,7 @@ "SOLR_URL": tasks.get_solr_url(SOLR_WRITER, COLLECTION), "ALMAOAI_LAST_HARVEST_FROM_DATE": CATALOG_LAST_HARVEST_FROM_DATE, "COMMAND": "ingest", + "DATA": "{{ ti.xcom_pull(task_ids='list_updated_files') | tojson }}", }}, trigger_rule="none_failed_min_one_success", dag=DAG @@ -217,6 +218,7 @@ "SOLR_AUTH_PASSWORD": SOLR_WRITER.password or "", "SOLR_URL": tasks.get_solr_url(SOLR_WRITER, COLLECTION), "COMMAND": "delete --suppress", + "DATA": "{{ ti.xcom_pull(task_ids='list_deleted_files') | tojson }}", }}, trigger_rule="none_failed_min_one_success", dag=DAG