Fix join on multiple destinations

teritorio · Apr 8, 2024 · 1327a97 · 1327a97
1 parent 4cfbd40
commit 1327a97
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/datasources/connectors/join.rb b/datasources/connectors/join.rb
@@ -31,13 +31,14 @@ def setup(kiba)
             'metadata' => JSON.parse(File.read("./#{@path}/#{source_config['metadata']}.metadata.json"))[source]
           })
         )
-      }
 
-      kiba.transform(JoinTransformer, JoinTransformer::Settings.from_hash({
-        'destination_id' => destination_id,
-        'key' => key,
-        'full_join' => true,
-      }))
+        kiba.transform(JoinTransformer, JoinTransformer::Settings.from_hash({
+          'source_ids' => [join.keys],
+          'destination_id' => destination_id,
+          'key' => key,
+          'full_join' => true,
+        }))
+      }
     }
   end
 end
diff --git a/datasources/transforms/join.rb b/datasources/transforms/join.rb
@@ -10,6 +10,7 @@ class JoinTransformer < Transformer
   extend T::Sig
 
   class Settings < Transformer::TransformerSettings
+    const :source_ids, T.nilable(T::Array[String])
     const :destination_id, String
     const :key, String
     const :full_join, T::Boolean, default: false
@@ -51,6 +52,8 @@ def process_tags(current_tags, update_tags, current_source, update_source)
   end
 
   def process_data(row)
+    return row if @settings.destination_id != row[:destination_id]
+
     key = JsonPath.on(row[:properties][:tags], @path)
     if key.present?
       if @rows.key?(key)