diff --git a/.changes/unreleased/Under the Hood-20241211-160216.yaml b/.changes/unreleased/Under the Hood-20241211-160216.yaml new file mode 100644 index 00000000000..8192dc9b303 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241211-160216.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Allow additional property on Model and SourceDefinition to avoid artifact read + problem +time: 2024-12-11T16:02:16.551106-08:00 +custom: + Author: ChenyuLInx + Issue: 8797 11123 diff --git a/core/dbt/artifacts/resources/v1/model.py b/core/dbt/artifacts/resources/v1/model.py index 9c43970f488..e80b88de9dd 100644 --- a/core/dbt/artifacts/resources/v1/model.py +++ b/core/dbt/artifacts/resources/v1/model.py @@ -10,6 +10,7 @@ ) from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.contracts.config.base import MergeBehavior +from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed from dbt_common.contracts.constraints import ModelLevelConstraint from dbt_common.dataclass_schema import dbtClassMixin @@ -35,7 +36,7 @@ class TimeSpine(dbtClassMixin): @dataclass -class Model(CompiledResource): +class Model(AdditionalPropertiesAllowed, CompiledResource): resource_type: Literal[NodeType.Model] access: AccessType = AccessType.Protected config: ModelConfig = field(default_factory=ModelConfig) diff --git a/core/dbt/artifacts/resources/v1/source_definition.py b/core/dbt/artifacts/resources/v1/source_definition.py index 9044307563e..2dbe1d2bb1d 100644 --- a/core/dbt/artifacts/resources/v1/source_definition.py +++ b/core/dbt/artifacts/resources/v1/source_definition.py @@ -56,7 +56,7 @@ class ParsedSourceMandatory(GraphResource, HasRelationMetadata): @dataclass -class SourceDefinition(ParsedSourceMandatory): +class SourceDefinition(AdditionalPropertiesAllowed, ParsedSourceMandatory): quoting: Quoting = field(default_factory=Quoting) loaded_at_field: Optional[str] = None freshness: Optional[FreshnessThreshold] = None diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 4bb70db5d9c..030ab783dfd 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -475,6 +475,8 @@ def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): del dct["_has_this"] if "previous_batch_results" in dct: del dct["previous_batch_results"] + if "batch" in dct: + del dct["batch"] return dct @classmethod @@ -1371,6 +1373,12 @@ def search_name(self): def group(self): return None + def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): + dct = super().__post_serialize__(dct, context) + if "_event_status" in dct: + del dct["_event_status"] + return dct + # ==================================== # Exposure node diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 5e9198b07f6..4ee1b3545a7 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -13,7 +13,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b4" + "default": "1.10.0a1" }, "generated_at": { "type": "string" @@ -4205,6 +4205,12 @@ }, "additionalProperties": false }, + "_extra": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "access": { "enum": [ "private", @@ -4779,7 +4785,7 @@ "default": null } }, - "additionalProperties": false, + "additionalProperties": true, "required": [ "database", "schema", @@ -7690,6 +7696,12 @@ "identifier": { "type": "string" }, + "_extra": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "quoting": { "type": "object", "title": "Quoting", @@ -8223,7 +8235,7 @@ "default": null } }, - "additionalProperties": false, + "additionalProperties": true, "required": [ "database", "schema", @@ -9121,19 +9133,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -14014,6 +14014,12 @@ }, "additionalProperties": false }, + "_extra": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "access": { "enum": [ "private", @@ -14588,7 +14594,7 @@ "default": null } }, - "additionalProperties": false, + "additionalProperties": true, "required": [ "database", "schema", @@ -17490,6 +17496,12 @@ "identifier": { "type": "string" }, + "_extra": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "quoting": { "type": "object", "title": "Quoting", @@ -18023,7 +18035,7 @@ "default": null } }, - "additionalProperties": false, + "additionalProperties": true, "required": [ "database", "schema", @@ -18712,19 +18724,7 @@ "type": "integer" }, "granularity": { - "enum": [ - "nanosecond", - "microsecond", - "millisecond", - "second", - "minute", - "hour", - "day", - "week", - "month", - "quarter", - "year" - ] + "type": "string" } }, "additionalProperties": false, @@ -20024,6 +20024,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -20178,6 +20199,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -20341,6 +20383,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -21586,6 +21649,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -21740,6 +21824,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -21903,6 +22008,27 @@ } ], "default": null + }, + "config": { + "anyOf": [ + { + "type": "object", + "title": "SemanticLayerElementConfig", + "properties": { + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, diff --git a/tests/unit/contracts/graph/test_manifest.py b/tests/unit/contracts/graph/test_manifest.py index 3505ee80037..0f3a80e5039 100644 --- a/tests/unit/contracts/graph/test_manifest.py +++ b/tests/unit/contracts/graph/test_manifest.py @@ -96,7 +96,6 @@ "deprecation_date", "defer_relation", "time_spine", - "batch", } ) diff --git a/tests/unit/contracts/graph/test_nodes.py b/tests/unit/contracts/graph/test_nodes.py index 3b509d0d20d..75184fb72cc 100644 --- a/tests/unit/contracts/graph/test_nodes.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -236,10 +236,11 @@ def test_basic_compiled_model(basic_compiled_dict, basic_compiled_model): assert node.is_ephemeral is False -def test_invalid_extra_fields_model(minimal_uncompiled_dict): - bad_extra = minimal_uncompiled_dict - bad_extra["notvalid"] = "nope" - assert_fails_validation(bad_extra, ModelNode) +def test_extra_fields_model_okay(minimal_uncompiled_dict): + extra = minimal_uncompiled_dict + extra["notvalid"] = "nope" + # Model still load fine with extra fields + assert ModelNode.from_dict(extra)._extra == {"notvalid": "nope"} def test_invalid_bad_type_model(minimal_uncompiled_dict): diff --git a/tests/unit/contracts/graph/test_nodes_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py index dc5a326f4d9..a8b1c917f6c 100644 --- a/tests/unit/contracts/graph/test_nodes_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -1950,7 +1950,6 @@ def test_basic_source_definition( node = basic_parsed_source_definition_object node_dict = basic_parsed_source_definition_dict minimum = minimum_parsed_source_definition_dict - assert_symmetric(node.to_resource(), node_dict, SourceDefinitionResource) assert node.is_ephemeral is False @@ -1961,6 +1960,13 @@ def test_basic_source_definition( pickle.loads(pickle.dumps(node)) +def test_extra_fields_source_definition_okay(minimum_parsed_source_definition_dict): + extra = minimum_parsed_source_definition_dict + extra["notvalid"] = "nope" + # Model still load fine with extra fields + assert SourceDefinition.from_dict(extra)._extra == {"notvalid": "nope"} + + def test_invalid_missing(minimum_parsed_source_definition_dict): bad_missing_name = minimum_parsed_source_definition_dict del bad_missing_name["name"] diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py index ec9cb57595d..76b6653d30a 100644 --- a/tests/unit/utils/__init__.py +++ b/tests/unit/utils/__init__.py @@ -197,11 +197,9 @@ def assert_from_dict(obj, dct, cls=None): cls.validate(dct) obj_from_dict = cls.from_dict(dct) - if hasattr(obj, "created_at"): obj_from_dict.created_at = 1 obj.created_at = 1 - assert obj_from_dict == obj