Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ADAP-774] fix snowflake_warehouse issue with dynamic tables #727

Merged
merged 31 commits into from
Aug 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
032b5b5
update RELEASE_BRANCH env
McKnight-42 Apr 14, 2023
16470c2
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Apr 25, 2023
4c475f1
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 May 25, 2023
81d5d5d
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 May 26, 2023
f1d8078
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 May 30, 2023
2773177
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 May 30, 2023
77a2b66
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 May 30, 2023
d9d3d2d
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 1, 2023
a51efc2
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 2, 2023
df68367
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 12, 2023
d9ab1ac
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 14, 2023
942473f
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 16, 2023
f579bf9
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 26, 2023
eb89108
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 28, 2023
2213e2c
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jun 29, 2023
a97950b
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 7, 2023
f5d4ebf
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 11, 2023
701a9ce
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 14, 2023
b462076
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 17, 2023
f57b14c
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 18, 2023
06083ab
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 19, 2023
508e7b2
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Jul 31, 2023
26b035c
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Aug 1, 2023
2f311e2
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Aug 3, 2023
fca0498
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Aug 4, 2023
cbb1ed2
Merge branch 'main' of gist.github.com:dbt-labs/dbt-snowflake
McKnight-42 Aug 8, 2023
aa8cb4f
start work on adap-774 to migrate dynmic tables to use snowflake_ware…
McKnight-42 Aug 8, 2023
2dc3819
revert some stuff back to wareshouse as it is what snwoflake expects,…
McKnight-42 Aug 9, 2023
a46cd9c
change location of comparion to snowflake_warehouse and warehouse
McKnight-42 Aug 9, 2023
0b7e464
remove uneeded addition
McKnight-42 Aug 9, 2023
a93bfe3
revert a field
McKnight-42 Aug 9, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions dbt/adapters/snowflake/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ def dynamic_table_config_changeset(
context=new_dynamic_table.target_lag,
)

if new_dynamic_table.warehouse != existing_dynamic_table.warehouse:
config_change_collection.warehouse = SnowflakeDynamicTableWarehouseConfigChange(
action=RelationConfigChangeAction.alter,
context=new_dynamic_table.warehouse,
if new_dynamic_table.snowflake_warehouse != existing_dynamic_table.snowflake_warehouse:
config_change_collection.snowflake_warehouse = (
SnowflakeDynamicTableWarehouseConfigChange(
action=RelationConfigChangeAction.alter,
context=new_dynamic_table.snowflake_warehouse,
)
)

if config_change_collection.has_changes:
Expand Down
18 changes: 10 additions & 8 deletions dbt/adapters/snowflake/relation_configs/dynamic_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class SnowflakeDynamicTableConfig(SnowflakeRelationConfigBase):
- name: name of the dynamic table
- query: the query behind the table
- target_lag: the maximum amount of time that the dynamic table’s content should lag behind updates to the base tables
- warehouse: the name of the warehouse that provides the compute resources for refreshing the dynamic table
- snowflake_warehouse: the name of the warehouse that provides the compute resources for refreshing the dynamic table

There are currently no non-configurable parameters.
"""
Expand All @@ -33,7 +33,7 @@ class SnowflakeDynamicTableConfig(SnowflakeRelationConfigBase):
database_name: str
query: str
target_lag: SnowflakeDynamicTableTargetLagConfig
warehouse: str
snowflake_warehouse: str

@classmethod
def from_dict(cls, config_dict) -> "SnowflakeDynamicTableConfig":
Expand All @@ -44,7 +44,7 @@ def from_dict(cls, config_dict) -> "SnowflakeDynamicTableConfig":
ComponentName.Database, config_dict.get("database_name")
),
"query": config_dict.get("query"),
"warehouse": config_dict.get("warehouse"),
"snowflake_warehouse": config_dict.get("snowflake_warehouse"),
}

if target_lag := config_dict.get("target_lag"):
Expand All @@ -62,7 +62,7 @@ def parse_model_node(cls, model_node: ModelNode) -> dict:
"schema_name": model_node.schema,
"database_name": model_node.database,
"query": model_node.compiled_code,
"warehouse": model_node.config.extra.get("snowflake_warehouse"),
"snowflake_warehouse": model_node.config.extra.get("snowflake_warehouse"),
}

if model_node.config.extra.get("target_lag"):
Expand All @@ -81,7 +81,7 @@ def parse_relation_results(cls, relation_results: RelationResults) -> dict:
"schema_name": dynamic_table.get("schema_name"),
"database_name": dynamic_table.get("database_name"),
"query": dynamic_table.get("text"),
"warehouse": dynamic_table.get("warehouse"),
"snowflake_warehouse": dynamic_table.get("warehouse"),
}

if dynamic_table.get("target_lag"):
Expand All @@ -108,17 +108,19 @@ def requires_full_refresh(self) -> bool:
@dataclass
class SnowflakeDynamicTableConfigChangeset:
target_lag: Optional[SnowflakeDynamicTableTargetLagConfigChange] = None
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
warehouse: Optional[SnowflakeDynamicTableWarehouseConfigChange] = None
snowflake_warehouse: Optional[SnowflakeDynamicTableWarehouseConfigChange] = None

@property
def requires_full_refresh(self) -> bool:
return any(
[
self.target_lag.requires_full_refresh if self.target_lag else False,
self.warehouse.requires_full_refresh if self.warehouse else False,
self.snowflake_warehouse.requires_full_refresh
if self.snowflake_warehouse
else False,
]
)

@property
def has_changes(self) -> bool:
return any([self.target_lag, self.warehouse])
return any([self.target_lag, self.snowflake_warehouse])
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@

{%- set target_lag = configuration_changes.target_lag -%}
{%- if target_lag -%}{{- log('Applying UPDATE TARGET_LAG to: ' ~ existing_relation) -}}{%- endif -%}
{%- set warehouse = configuration_changes.warehouse -%}
{%- if warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%}
{%- set snowflake_warehouse = configuration_changes.snowflake_warehouse -%}
{%- if snowflake_warehouse -%}{{- log('Applying UPDATE WAREHOUSE to: ' ~ existing_relation) -}}{%- endif -%}

alter dynamic table {{ existing_relation }} set
{% if target_lag %}target_lag = '{{ target_lag.context }}'{% endif %}
{% if warehouse %}warehouse = {{ warehouse.context }}{% endif %}
{% if snowflake_warehouse %}warehouse = {{ snowflake_warehouse.context }}{% endif %}

{%- endif -%}

Expand Down