Edited template to consider metadata tag from tables.yml file

Cargill · Oct 31, 2018 · e7cb3d3 · e7cb3d3
1 parent 86d6b14
commit e7cb3d3
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 17 deletions.
diff --git a/templates/sqoop-parquet-hdfs-hive-merge/create-base-table.sql b/templates/sqoop-parquet-hdfs-hive-merge/create-base-table.sql
@@ -22,12 +22,12 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {{ table.destination.name }}_base (
   {%- endfor %})
 STORED AS Parquet
 LOCATION '{{ conf.staging_database.path }}/{{ table.destination.name }}/base'
+{%- if table.metadata %}
 TBLPROPERTIES(
-  'parquet.compression'='SNAPPY',
-  'SOURCE' = '{{ table.META_SOURCE }}',
-  'SECURITY_CLASSIFICATION' = '{{ table.META_SECURITY_CLASSIFICATION }}',
-  'LOAD_FREQUENCY' = '{{ table.META_LOAD_FREQUENCY }}',
-  'CONTACT_INFO' = '{{ table.META_CONTACT_INFO }}'
-);
+  {%- for key, value in table.metadata.items() %}
+   '{{ key }}' = '{{ value }}'{%- if not loop.last -%}, {% endif %}
+  {%- endfor %}
+)
+{%- endif %};
 
 COMPUTE STATS {{ table.destination.name }}_base
diff --git a/templates/sqoop-parquet-hdfs-hive-merge/create-incr-table.sql b/templates/sqoop-parquet-hdfs-hive-merge/create-incr-table.sql
@@ -23,11 +23,12 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {{ table.destination.name }}_incr (
   {%- endfor %})
 STORED AS Parquet
 LOCATION '{{ conf.staging_database.path }}/{{ table.destination.name }}/incr'
+{%- if table.metadata %}
 TBLPROPERTIES(
-  'SOURCE' = '{{ table.META_SOURCE }}',
-  'SECURITY_CLASSIFICATION' = '{{ table.META_SECURITY_CLASSIFICATION }}',
-  'LOAD_FREQUENCY' = '{{ table.META_LOAD_FREQUENCY }}',
-  'CONTACT_INFO' = '{{ table.META_CONTACT_INFO }}'
-);
+  {%- for key, value in table.metadata.items() %}
+   '{{ key }}' = '{{ value }}'{%- if not loop.last -%}, {% endif %}
+  {%- endfor %}
+)
+{%- endif %};
 
 COMPUTE STATS {{ table.destination.name }}_incr
diff --git a/templates/sqoop-parquet-hdfs-hive-merge/create-report-table.sql b/templates/sqoop-parquet-hdfs-hive-merge/create-report-table.sql
@@ -35,13 +35,13 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {{ table.destination.name }}_report (
   {%- endfor %})
 STORED AS Parquet
 LOCATION '{{ conf.staging_database.path }}/{{ table.destination.name }}/base'
+{%- if table.metadata %}
 TBLPROPERTIES(
-  'parquet.compression'='SNAPPY',
-  'SOURCE' = '{{ table.META_SOURCE }}',
-  'SECURITY_CLASSIFICATION' = '{{ table.META_SECURITY_CLASSIFICATION }}',
-  'LOAD_FREQUENCY' = '{{ table.META_LOAD_FREQUENCY }}',
-  'CONTACT_INFO' = '{{ table.META_CONTACT_INFO }}'
-);
+  {%- for key, value in table.metadata.items() %}
+   '{{ key }}' = '{{ value }}'{%- if not loop.last -%}, {% endif %}
+  {%- endfor %}
+)
+{%- endif %};
 
 insert overwrite {{ table.destination.name }}_report select * from {{ table.destination.name }}_merge_view;