From 0e02477b3d8b241486961ef5808ad50c24d0d931 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 28 Jun 2024 18:56:51 -0400 Subject: [PATCH 1/5] fix get_columns_in_relation --- dbt/include/snowflake/macros/adapters.sql | 4 ++-- tests/functional/adapter/empty/_models.py | 19 ++++++++++++++++++ tests/functional/adapter/empty/test_empty.py | 21 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/empty/_models.py diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index b858aec11..a88e011ee 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -27,14 +27,14 @@ {% macro snowflake__get_columns_in_relation(relation) -%} {%- set sql -%} - describe table {{ relation }} + describe table {{ relation.render() }} {%- endset -%} {%- set result = run_query(sql) -%} {% set maximum = 10000 %} {% if (result | length) >= maximum %} {% set msg %} - Too many columns in relation {{ relation }}! dbt can only get + Too many columns in relation {{ relation.render() }}! dbt can only get information about relations with fewer than {{ maximum }} columns. {% endset %} {% do exceptions.raise_compiler_error(msg) %} diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py new file mode 100644 index 000000000..4fa84c7fb --- /dev/null +++ b/tests/functional/adapter/empty/_models.py @@ -0,0 +1,19 @@ +SEED = """ +id,value +1,a +2,b +3,c +""".strip() + + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ + config(materialized="table") +}} +select {{ adapter.get_columns_in_relation(ref("my_seed"))|length }} as columns +""" diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 401179b71..f6dac6099 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -1,4 +1,8 @@ from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef +from dbt.tests.util import run_dbt +import pytest + +from tests.functional.adapter.empty import _models class TestSnowflakeEmpty(BaseTestEmpty): @@ -7,3 +11,20 @@ class TestSnowflakeEmpty(BaseTestEmpty): class TestSnowflakeEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): pass + + +class TestMetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + } + + def test_run(self, project): + run_dbt(["seed"]) + run_dbt(["run", "--empty"]) From c259215a3f141bdd8d82aea746ac0e53b21601e0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 28 Jun 2024 19:01:46 -0400 Subject: [PATCH 2/5] changelog --- .changes/unreleased/Fixes-20240628-190140.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240628-190140.yaml diff --git a/.changes/unreleased/Fixes-20240628-190140.yaml b/.changes/unreleased/Fixes-20240628-190140.yaml new file mode 100644 index 000000000..c58b465fd --- /dev/null +++ b/.changes/unreleased/Fixes-20240628-190140.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix scenario where using the `--empty` flag causes metadata queries to contain + limit clauses +time: 2024-06-28T19:01:40.558234-04:00 +custom: + Author: mikealfare + Issue: "1033" From 6d6bc21983552a08afbcc33124da826c68c53aec Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 2 Jul 2024 00:31:04 -0400 Subject: [PATCH 3/5] add render to relation in metadata queries --- dbt/include/snowflake/macros/adapters.sql | 12 ++++++------ tests/functional/adapter/empty/_models.py | 15 ++++++++++----- tests/functional/adapter/empty/test_empty.py | 16 ++++++++++++++-- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index a88e011ee..177720486 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -186,7 +186,7 @@ {% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%} {% call statement('alter_column_type') %} - alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; + alter table {{ relation.render() }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }}; {% endcall %} {% endmacro %} @@ -196,7 +196,7 @@ {%- else -%} {%- set relation_type = relation.type -%} {%- endif -%} - comment on {{ relation_type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$; + comment on {{ relation_type }} {{ relation.render() }} IS $${{ relation_comment | replace('$', '[$]') }}$$; {% endmacro %} @@ -207,7 +207,7 @@ {% else -%} {% set relation_type = relation.type %} {% endif %} - alter {{ relation_type }} {{ relation }} alter + alter {{ relation_type }} {{ relation.render() }} alter {% for column_name in existing_columns if (column_name in existing_columns) or (column_name|lower in existing_columns) %} {{ get_column_comment_sql(column_name, column_dict) }} {{- ',' if not loop.last else ';' }} {% endfor %} @@ -266,7 +266,7 @@ {% if add_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} add column + alter {{ relation_type }} {{ relation.render() }} add column {% for column in add_columns %} {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} @@ -279,7 +279,7 @@ {% if remove_columns %} {% set sql -%} - alter {{ relation_type }} {{ relation }} drop column + alter {{ relation_type }} {{ relation.render() }} drop column {% for column in remove_columns %} {{ column.name }}{{ ',' if not loop.last }} {% endfor %} @@ -312,7 +312,7 @@ {% macro snowflake__truncate_relation(relation) -%} {% set truncate_dml %} - truncate table {{ relation }} + truncate table {{ relation.render() }} {% endset %} {% call statement('truncate_relation') -%} {{ snowflake_dml_explicit_transaction(truncate_dml) }} diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py index 4fa84c7fb..338ad221d 100644 --- a/tests/functional/adapter/empty/_models.py +++ b/tests/functional/adapter/empty/_models.py @@ -1,5 +1,5 @@ SEED = """ -id,value +my_id,my_value 1,a 2,b 3,c @@ -12,8 +12,13 @@ GET_COLUMNS_IN_RELATION = """ -{{ - config(materialized="table") -}} -select {{ adapter.get_columns_in_relation(ref("my_seed"))|length }} as columns +{{ config(materialized="table") }} +select {{ adapter.get_columns_in_relation(ref("my_seed"))|length }} as get_columns_in_relation +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "string") }} +select * from {{ ref("my_seed") }} """ diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index f6dac6099..7cc90b0af 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -23,8 +23,20 @@ def models(self): return { "control.sql": _models.CONTROL, "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, } - def test_run(self, project): + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): run_dbt(["seed"]) - run_dbt(["run", "--empty"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) From a4042d498ee7127ed5011ecc86b3c521bc835e26 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 2 Jul 2024 10:32:06 -0400 Subject: [PATCH 4/5] add tests for all render instances --- tests/functional/adapter/empty/_models.py | 50 +++++++++++++++++++- tests/functional/adapter/empty/test_empty.py | 9 ++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py index 338ad221d..e0420f84a 100644 --- a/tests/functional/adapter/empty/_models.py +++ b/tests/functional/adapter/empty/_models.py @@ -6,6 +6,17 @@ """.strip() +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + CONTROL = """ select * from {{ ref("my_seed") }} """ @@ -13,7 +24,8 @@ GET_COLUMNS_IN_RELATION = """ {{ config(materialized="table") }} -select {{ adapter.get_columns_in_relation(ref("my_seed"))|length }} as get_columns_in_relation +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} """ @@ -22,3 +34,39 @@ {{ alter_column_type(ref("my_seed"), "MY_VALUE", "string") }} select * from {{ ref("my_seed") }} """ + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/tests/functional/adapter/empty/test_empty.py b/tests/functional/adapter/empty/test_empty.py index 7cc90b0af..fe07fc081 100644 --- a/tests/functional/adapter/empty/test_empty.py +++ b/tests/functional/adapter/empty/test_empty.py @@ -21,9 +21,14 @@ def seeds(self): @pytest.fixture(scope="class") def models(self): return { + "schema.yml": _models.SCHEMA, "control.sql": _models.CONTROL, "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, } @pytest.fixture(scope="class", autouse=True) @@ -36,6 +41,10 @@ def setup(self, project): "control", "get_columns_in_relation", "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", ], ) def test_run(self, project, model): From 799e5959f0863743fd8b7a2fe8094b7382b19e09 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 2 Jul 2024 15:49:26 -0400 Subject: [PATCH 5/5] update alter_column_type test to use more generic varchar --- tests/functional/adapter/empty/_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/empty/_models.py b/tests/functional/adapter/empty/_models.py index e0420f84a..698e4f1db 100644 --- a/tests/functional/adapter/empty/_models.py +++ b/tests/functional/adapter/empty/_models.py @@ -31,7 +31,7 @@ ALTER_COLUMN_TYPE = """ {{ config(materialized="table") }} -{{ alter_column_type(ref("my_seed"), "MY_VALUE", "string") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} select * from {{ ref("my_seed") }} """