-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[DP-2782] Fix data product assets issue and bump the version (#3040)
* Don't import openmetadata by default It's incompatable with python 3.11 which is really annoying. * Fix bug appending assets to a data product The list append() method returns None, so this was clearing the asset list every other time an asset was added. * Bump version
- Loading branch information
Showing
9 changed files
with
314 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
1 change: 0 additions & 1 deletion
1
python-libraries/data-platform-catalogue/data_platform_catalogue/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,4 @@ | ||
| from .client import DataHubCatalogueClient # noqa: F401 | ||
| from .client import OpenMetadataCatalogueClient # noqa: F401 | ||
| from .client import CatalogueError, ReferencedEntityMissing # noqa: F401 | ||
| from .entities import DataProductMetadata # noqa: F401 | ||
| from .entities import CatalogueMetadata, DataLocation, TableMetadata # noqa: F401 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
256 changes: 256 additions & 0 deletions
256
...ries/data-platform-catalogue/tests/snapshots/datahub_create_two_tables_with_metadata.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,256 @@ | ||
| [ | ||
| { | ||
| "entityType": "dataset", | ||
| "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table,PROD)", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "datasetProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": {}, | ||
| "description": "bla bla", | ||
| "tags": [] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataset", | ||
| "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table,PROD)", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "schemaMetadata", | ||
| "aspect": { | ||
| "json": { | ||
| "schemaName": "my_table", | ||
| "platform": "urn:li:dataPlatform:glue", | ||
| "version": 1, | ||
| "created": { | ||
| "time": 0, | ||
| "actor": "urn:li:corpuser:unknown" | ||
| }, | ||
| "lastModified": { | ||
| "time": 0, | ||
| "actor": "urn:li:corpuser:unknown" | ||
| }, | ||
| "hash": "", | ||
| "platformSchema": { | ||
| "com.linkedin.schema.OtherSchema": { | ||
| "rawSchema": "" | ||
| } | ||
| }, | ||
| "fields": [ | ||
| { | ||
| "fieldPath": "foo", | ||
| "nullable": false, | ||
| "description": "a", | ||
| "type": { | ||
| "type": { | ||
| "com.linkedin.schema.StringType": {} | ||
| } | ||
| }, | ||
| "nativeDataType": "string", | ||
| "recursive": false, | ||
| "isPartOfKey": false | ||
| }, | ||
| { | ||
| "fieldPath": "bar", | ||
| "nullable": false, | ||
| "description": "b", | ||
| "type": { | ||
| "type": { | ||
| "com.linkedin.schema.NumberType": {} | ||
| } | ||
| }, | ||
| "nativeDataType": "int", | ||
| "recursive": false, | ||
| "isPartOfKey": false | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "domain", | ||
| "entityUrn": "urn:li:domain:legal-aid", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "domainProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "name": "legal-aid", | ||
| "description": "" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "domains", | ||
| "aspect": { | ||
| "json": { | ||
| "domains": [ | ||
| "urn:li:domain:legal-aid" | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "dataProductProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": { | ||
| "email": "justice@justice.gov.uk", | ||
| "retention_period_in_days": "365", | ||
| "dpia_required": "False" | ||
| }, | ||
| "name": "my_data_product", | ||
| "description": "bla bla" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "dataProductProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": {}, | ||
| "assets": [ | ||
| { | ||
| "sourceUrn": "urn:li:dataProduct:my_data_product", | ||
| "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table,PROD)" | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataset", | ||
| "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table2,PROD)", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "datasetProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": {}, | ||
| "description": "this is a different table", | ||
| "tags": [] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataset", | ||
| "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table2,PROD)", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "schemaMetadata", | ||
| "aspect": { | ||
| "json": { | ||
| "schemaName": "my_table2", | ||
| "platform": "urn:li:dataPlatform:glue", | ||
| "version": 1, | ||
| "created": { | ||
| "time": 0, | ||
| "actor": "urn:li:corpuser:unknown" | ||
| }, | ||
| "lastModified": { | ||
| "time": 0, | ||
| "actor": "urn:li:corpuser:unknown" | ||
| }, | ||
| "hash": "", | ||
| "platformSchema": { | ||
| "com.linkedin.schema.OtherSchema": { | ||
| "rawSchema": "" | ||
| } | ||
| }, | ||
| "fields": [ | ||
| { | ||
| "fieldPath": "boo", | ||
| "nullable": false, | ||
| "description": "spooky", | ||
| "type": { | ||
| "type": { | ||
| "com.linkedin.schema.BooleanType": {} | ||
| } | ||
| }, | ||
| "nativeDataType": "boolean", | ||
| "recursive": false, | ||
| "isPartOfKey": false | ||
| }, | ||
| { | ||
| "fieldPath": "yar", | ||
| "nullable": false, | ||
| "description": "shiver my timbers", | ||
| "type": { | ||
| "type": { | ||
| "com.linkedin.schema.StringType": {} | ||
| } | ||
| }, | ||
| "nativeDataType": "string", | ||
| "recursive": false, | ||
| "isPartOfKey": false | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "domain", | ||
| "entityUrn": "urn:li:domain:legal-aid", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "domainProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "name": "legal-aid", | ||
| "description": "" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "domains", | ||
| "aspect": { | ||
| "json": { | ||
| "domains": [ | ||
| "urn:li:domain:legal-aid" | ||
| ] | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "dataProductProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": { | ||
| "email": "justice@justice.gov.uk", | ||
| "retention_period_in_days": "365", | ||
| "dpia_required": "False" | ||
| }, | ||
| "name": "my_data_product", | ||
| "description": "bla bla" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "entityType": "dataproduct", | ||
| "entityUrn": "urn:li:dataProduct:my_data_product", | ||
| "changeType": "UPSERT", | ||
| "aspectName": "dataProductProperties", | ||
| "aspect": { | ||
| "json": { | ||
| "customProperties": {}, | ||
| "assets": [ | ||
| { | ||
| "sourceUrn": "urn:li:dataProduct:my_data_product", | ||
| "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,my_database.my_table2,PROD)" | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| } | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 2 additions & 4 deletions
6
python-libraries/data-platform-catalogue/tests/test_client_openmetadata.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters