diff --git a/Makefile b/Makefile index 4c23f3266..c2bda2e0c 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ spark-test: engine-spark-up pytest -n auto -x -m "spark or pyspark" --retries 3 --junitxml=test-results/junit-spark.xml trino-test: engine-trino-up - pytest -n auto -x -m "trino or trino_iceberg or trino_delta" --retries 3 --junitxml=test-results/junit-trino.xml + pytest -n auto -x -m "trino or trino_iceberg or trino_delta or trino_nessie" --retries 3 --junitxml=test-results/junit-trino.xml ################# # Cloud Engines # diff --git a/docs/integrations/engines/trino.md b/docs/integrations/engines/trino.md index 6802d93bd..2dd46625c 100644 --- a/docs/integrations/engines/trino.md +++ b/docs/integrations/engines/trino.md @@ -47,7 +47,11 @@ iceberg.catalog.type=hive_metastore **Note**: The Trino Iceberg Connector must be configured with an `iceberg.catalog.type` that supports views. At the time of this writing, this is `hive_metastore`, `glue`, and `rest`. -The `jdbc` and `nessie` catalogs do not support views and are thus incompatible with SQLMesh. +The `jdbc` and `nessie` iceberg catalog types do not support views and are thus incompatible with SQLMesh. + +!!! info "Nessie" + Nessie is supported when used as an Iceberg REST Catalog (`iceberg.catalog.type=rest`). + For more information on how to configure the Trino Iceberg connector for this, see the [Nessie documentation](https://projectnessie.org/nessie-latest/trino/). #### Delta Lake Connector Configuration diff --git a/tests/core/engine_adapter/integration/config.yaml b/tests/core/engine_adapter/integration/config.yaml index 77f63d41f..1a64aba0b 100644 --- a/tests/core/engine_adapter/integration/config.yaml +++ b/tests/core/engine_adapter/integration/config.yaml @@ -41,6 +41,17 @@ gateways: retries: 20 state_connection: type: duckdb + inttest_trino_nessie: + connection: + type: trino + host: {{ env_var('DOCKER_HOSTNAME', 'localhost') }} + port: 8080 + user: admin + catalog: datalake_nessie + http_scheme: http + retries: 20 + state_connection: + type: duckdb inttest_spark: connection: type: spark diff --git a/tests/core/engine_adapter/integration/docker/_common-hive.yaml b/tests/core/engine_adapter/integration/docker/_common-hive.yaml index 657d7e296..59d3cd80a 100644 --- a/tests/core/engine_adapter/integration/docker/_common-hive.yaml +++ b/tests/core/engine_adapter/integration/docker/_common-hive.yaml @@ -35,7 +35,8 @@ services: /usr/bin/mc mb --quiet myminio/trino/testing_iceberg; /usr/bin/mc mb --quiet myminio/trino/testing_delta; /usr/bin/mc mb --quiet myminio/spark/datalake; - /usr/bin/mc mb --quiet myminio/spark/testing + /usr/bin/mc mb --quiet myminio/spark/testing; + /usr/bin/mc mb --quiet myminio/nessie/warehouse; " depends_on: - minio \ No newline at end of file diff --git a/tests/core/engine_adapter/integration/docker/compose.trino.yaml b/tests/core/engine_adapter/integration/docker/compose.trino.yaml index a8c59cee7..a3d904b8d 100644 --- a/tests/core/engine_adapter/integration/docker/compose.trino.yaml +++ b/tests/core/engine_adapter/integration/docker/compose.trino.yaml @@ -83,4 +83,28 @@ services: HIVE_METASTORE_WAREHOUSE_DIR: s3://trino/datalake_delta <<: *hive_metastore_environments depends_on: - - metastore \ No newline at end of file + - metastore + + nessie: + image: ghcr.io/projectnessie/nessie:0.102.2 + restart: on-failure + ports: + - '19120:19120' + environment: + nessie.version.store.type: JDBC2 + nessie.version.store.persist.jdbc.datasource: postgresql + quarkus.datasource.postgresql.jdbc.url: jdbc:postgresql://metastore:5432/nessie + quarkus.datasource.postgresql.username: hive + quarkus.datasource.postgresql.password: hive + nessie.catalog.default-warehouse: warehouse + nessie.catalog.warehouses.warehouse.location: s3://nessie/warehouse + nessie.catalog.service.s3.default-options.region: us-east-1 + nessie.catalog.service.s3.default-options.path-style-access: 'true' + nessie.catalog.service.s3.default-options.access-key: urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key + nessie.catalog.secrets.access-key.name: minio + nessie.catalog.secrets.access-key.secret: minio123 + nessie.catalog.service.s3.default-options.endpoint: http://minio:9000/ + + depends_on: + - metastore + diff --git a/tests/core/engine_adapter/integration/docker/init-metastore-db.sql b/tests/core/engine_adapter/integration/docker/init-metastore-db.sql index 3535d60e0..dd8467083 100644 --- a/tests/core/engine_adapter/integration/docker/init-metastore-db.sql +++ b/tests/core/engine_adapter/integration/docker/init-metastore-db.sql @@ -10,4 +10,5 @@ create database datalake_iceberg_metastore; create database datalake_delta_metastore; create database testing_metastore; create database testing_iceberg_metastore; -create database testing_delta_metastore; \ No newline at end of file +create database testing_delta_metastore; +create database nessie; \ No newline at end of file diff --git a/tests/core/engine_adapter/integration/docker/trino/catalog/datalake_nessie.properties b/tests/core/engine_adapter/integration/docker/trino/catalog/datalake_nessie.properties new file mode 100644 index 000000000..6ac178625 --- /dev/null +++ b/tests/core/engine_adapter/integration/docker/trino/catalog/datalake_nessie.properties @@ -0,0 +1,14 @@ +connector.name=iceberg +iceberg.catalog.type=rest +iceberg.rest-catalog.security=NONE +iceberg.rest-catalog.uri=http://nessie:19120/iceberg/ +iceberg.rest-catalog.vended-credentials-enabled=false +iceberg.metadata-cache.enabled=false +iceberg.hive-catalog-name=datalake + +fs.native-s3.enabled=true +s3.endpoint=http://minio:9000 +s3.path-style-access=true +s3.aws-access-key=minio +s3.aws-secret-key=minio123 +s3.region=us-east-1 \ No newline at end of file diff --git a/tests/core/engine_adapter/integration/test_integration.py b/tests/core/engine_adapter/integration/test_integration.py index 4e4f4a405..614bd6c7b 100644 --- a/tests/core/engine_adapter/integration/test_integration.py +++ b/tests/core/engine_adapter/integration/test_integration.py @@ -146,6 +146,14 @@ def test_type(request): pytest.mark.trino_delta, ], ), + pytest.param( + "trino_nessie", + marks=[ + pytest.mark.docker, + pytest.mark.engine, + pytest.mark.trino_nessie, + ], + ), pytest.param( "spark", marks=[