Sync integration tests with main

backend/dataall/core/stacks/api/types.py

@@ -20,6 +20,7 @@
         gql.Field(name='region', type=gql.NonNullableType(gql.String)),
         gql.Field(name='status', type=gql.String),
         gql.Field(name='stackid', type=gql.String),
+        gql.Field(name='updated', type=gql.AWSDateTime),
         gql.Field(name='link', type=gql.String, resolver=resolve_link),
         gql.Field(name='outputs', type=gql.String, resolver=resolve_outputs),
         gql.Field(name='resources', type=gql.String, resolver=resolve_resources),

tests_new/integration_tests/README.md

@@ -10,60 +10,154 @@ Currently **we support only Cognito based deployments** but support for any IdP
 
 ## Pre-requisites
 
-- A real deployment of data.all in AWS
-- An SSM parameter (`/{resource_prefix/{env_name}/testdata`) with the following contents
-    ```
-    {
-      "users": {
-        "testUserTenant": {
-          "username": "testUserTenant",
-          "password": "...",
-          "groups": [
-            "DAAdministrators"
-          ]
-        },
-        "testUser1": {
-          "username": "testUser1",
-          "password": "...",
-          "groups": [
-            "testGroup1"
-          ]
-        },
-        "testUser2": {
-          "username": "testUser2",
-          "password": "...",
-          "groups": [
-            "testGroup2"
-          ]
-        },
-        "testUser3": {
-          "username": "testUser3",
-          "password": "...",
-          "groups": [
-            "testGroup3"
-          ]
-        },
-        "testUser4": {
-          "username": "testUser4",
-          "password": "...",
-          "groups": [
-            "testGroup4"
-          ]
-        }
-      },
-      "envs": {
-        "session_env1": {
-          "accountId": "...",
-          "region": "eu-central-1"
-        },
-        "session_env2": {
-          "accountId": "...",
-          "region": "eu-west-1"
-        }
-      }
-    }
-    ```
+- A real deployment of data.all in AWS. 
+     - For this deployment the `cdk.json` flag `enable_pivot_role_auto_create` must be set to `true`.
+       - For this deployment the `config.json` flag `cdk_pivot_role_multiple_environments_same_account` must be set to `true` if an AWS account is going to be reused for multiple environments,
+         - Second test account is bootstraped, and first account is added to trusted policy in target regions
+          ```cdk bootstrap --trust <first-account-id> -c @aws-cdk/core:newStyleStackSynthesis=true --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess aws://<second-account-id>/region```
+         - An SSM parameter (`/dataall/{env_name}/testdata`) in the DEPLOYMENT ACCOUNT with the following contents
+          ```
+          {
+            "users": {
+              "testUserTenant": {
+                "username": "testUserTenant",
+                "password": "...",
+                "groups": [
+                  "DAAdministrators"
+                ]
+              },
+              "testUser1": {
+                "username": "testUser1",
+                "password": "...",
+                "groups": [
+                  "testGroup1"
+                ]
+              },
+              "testUser2": {
+                "username": "testUser2",
+                "password": "...",
+                "groups": [
+                  "testGroup2"
+                ]
+              },
+              "testUser3": {
+                "username": "testUser3",
+                "password": "...",
+                "groups": [
+                  "testGroup3"
+                ]
+              },
+              "testUser4": {
+                "username": "testUser4",
+                "password": "...",
+                "groups": [
+                  "testGroup4"
+                ]
+              }
+            },
+              "envs": {
+              "session_env1": {
+                "accountId": "...",
+                "region": "eu-central-1"
+              },
+              "session_env2": {
+                "accountId": "...",
+                "region": "eu-west-1"
+              },
+              "persistent_env1": {
+                "accountId": "...",
+                "region": "us-east-1"
+              },
+               "persistent_cross_acc_env_1": {
+                  "accountId": "...",
+                  "region": "us-east-1"
+                },
+              "session_cross_acc_env_1": {
+                "accountId": "...",
+                "region": "eu-central-1"
+              }
+            },
+            "dashboards": {
+                "session_env1": {
+                  "dashboardId": "..."
+                },
+              },
+             "redshift_connections": {
+              "connection_serverless_admin_session_env1": {
+                "namespace_id": "...",
+                "workgroup": "...",
+                "secret_arn": "..."
+              },
+              "connection_serverless_data_user_session_env1": {
+                "namespace_id": "...",
+                "workgroup": "...",
+                "secret_arn": "..."
+              },
+              "connection_cluster_admin_session_cross_acc_env_1": {
+                "cluster_id": "...",
+                "secret_arn": "..."
+              },
+              "connection_cluster_data_user_session_cross_acc_env_1": {
+                "cluster_id": "...",
+                "secret_arn": "..."
+              }
+            }
+          }
+          ```
+
 - The pipeline will create the users/groups
+- For Redshift testing we require some pre-existing infrastructure:
+  - One Redshift serverless namespace+workgroup deployed in `session_env1` and one Redshift provisioned cluster in `session_cross_acc_env_1` 
+  - The provisioned cluster MUST be encrypted and use RA3 cluster type (Check the [docs](https://docs.aws.amazon.com/redshift/latest/dg/datashare-overview.html) for other data sharing limitations)
+  - Both clusters must host the default `dev` database with the `public` schema.
+  - For each we need to ensure that the admin credentials are stored in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}. If you are going to use the Redshift Query Editor, then you will also need the tag {key:Redshift, value:any}
+  - For each we need to create a Redshift user (see SQL commands below) and store the credentials in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}. If you are going to use the Redshift Query Editor, then you will also need the tag {key:Redshift, value:any}
+  - For each we need to create a set of tables using the commands below
+  - For each we need to create a Redshift role as in the commands below
+
+Create User and grant basic permissions using admin connection
+```sql
+CREATE USER testuser PASSWORD 'Pass1Word!';
+GRANT USAGE ON SCHEMA public TO testuser;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO testuser;
+```
+
+Create and attach role using admin connection
+```sql
+CREATE ROLE testrole;
+GRANT ROLE testrole TO testuser;
+```
+
+Create tables using testuser connection
+```sql
+DROP TABLE IF EXISTS nation;
+DROP TABLE IF EXISTS region;
+
+CREATE TABLE region (
+  R_REGIONKEY bigint NOT NULL,
+  R_NAME varchar(25),
+  R_COMMENT varchar(152))
+diststyle all;
+
+CREATE TABLE nation (
+  N_NATIONKEY bigint NOT NULL,
+  N_NAME varchar(25),
+  N_REGIONKEY bigint,
+  N_COMMENT varchar(152))
+diststyle all;
+```
+
+### Dashboard Tests Pre-Requisities
+
+In order to run the tests on the dashboards module the following steps are required:
+
+- Create Enterprise QuickSight Subscription in `session_env1` AWS Account
+- Update QuickSight Account with a Reader Capacity Pricing Plan (required for generating embed URLs - `GenerateEmbedUrlForAnonymousUser`)
+- Create / Publish a QuickSight Dashboard
+- Create a QuickSight Group named `dataall` and give owner access of the published dashboard to the `dataall` group
+- Provide the `dashboardId` in the `config.json` as shown above
+
+Rather than failing if the above pre-requisites are not completed, if ther eis no QuickSight Account is detected in `session_env1` the dashboard tests will be **skipped**.
 
 ## Run tests
 
@@ -88,4 +182,9 @@ You can also run the tests locally by...
 
 ## Coverage
 
-At the moment integration tests only cover Organizations module as an example.
+At the moment integration tests cover:
+- Organizations
+- Environments
+- S3 Datasets
+- Notebooks
+- Worksheets

tests_new/integration_tests/aws_clients/s3.py

@@ -0,0 +1,36 @@
+import logging
+from botocore.exceptions import ClientError
+
+log = logging.getLogger(__name__)
+
+
+class S3Client:
+    def __init__(self, session, account, region):
+        self._client = session.client('s3', region_name=region)
+        self._control_client = session.client('s3control', region_name=region)
+        self._resource = session.resource('s3', region_name=region)
+        self._account = account
+        self._region = region
+
+    def delete_bucket(self, bucket_name):
+        """
+        Delete an S3 bucket.
+        :param bucket_name: Name of the S3 bucket to be deleted
+        :return: None
+        """
+        try:
+            # Delete all objects in the bucket before deleting the bucket
+            bucket = self._resource.Bucket(bucket_name)
+            # Delete all object versions
+            bucket.object_versions.all().delete()
+            # Delete any remaining objects (if versioning was not enabled)
+            bucket.objects.all().delete()
+            # Delete any remaining access point
+            access_points = self._control_client.list_access_points(AccountId=self._account, Bucket=bucket_name)[
+                'AccessPointList'
+            ]
+            for access_point in access_points:
+                self._control_client.delete_access_point(AccountId=self._account, Name=access_point['Name'])
+            bucket.delete()
+        except ClientError as e:
+            log.exception(f'Error deleting S3 bucket: {e}')

tests_new/integration_tests/aws_clients/sts.py

@@ -0,0 +1,57 @@
+from uuid import uuid4
+import boto3
+from boto3 import Session
+from botocore.credentials import RefreshableCredentials
+from botocore.session import get_session
+
+SESSION_EXPIRATION_TIME_IN_SECONDS = 3600
+
+
+class STSClient:
+    def __init__(self, role_arn, region, session_name=None):
+        self.role_arn = role_arn
+        self.region = region
+        self.session_name = session_name or uuid4().hex
+
+    def _refresh_credentials(self):
+        params = {
+            'RoleArn': self.role_arn,
+            'RoleSessionName': self.session_name,
+            'DurationSeconds': SESSION_EXPIRATION_TIME_IN_SECONDS,
+        }
+        sts_client = boto3.client('sts', region_name=self.region)
+
+        response = sts_client.assume_role(**params).get('Credentials')
+        credentials = {
+            'access_key': response.get('AccessKeyId'),
+            'secret_key': response.get('SecretAccessKey'),
+            'token': response.get('SessionToken'),
+            'expiry_time': response.get('Expiration').isoformat(),
+        }
+        return credentials
+
+    def get_refreshable_session(self) -> Session:
+        """
+        Get refreshable boto3 session.
+        """
+        refreshable_credentials = RefreshableCredentials.create_from_metadata(
+            metadata=self._refresh_credentials(),
+            refresh_using=self._refresh_credentials,
+            method='sts-assume-role',
+        )
+
+        session = get_session()
+        session._credentials = refreshable_credentials
+        session.set_config_variable('region', self.region)
+        return Session(botocore_session=session)
+
+    def get_role_session(self, session) -> Session:
+        sts_client = session.client('sts', region_name=self.region)
+        assumed_role_object = sts_client.assume_role(RoleArn=self.role_arn, RoleSessionName=self.session_name)
+        credentials = assumed_role_object['Credentials']
+
+        return boto3.Session(
+            aws_access_key_id=credentials['AccessKeyId'],
+            aws_secret_access_key=credentials['SecretAccessKey'],
+            aws_session_token=credentials['SessionToken'],
+        )

tests_new/integration_tests/client.py

@@ -1,42 +1,49 @@
-import requests
+import logging
 import os
 import uuid
+from pprint import pformat
 from urllib.parse import parse_qs, urlparse
+
+import requests
 from munch import DefaultMunch
-from retrying import retry
-from integration_tests.errors import GqlError
 from oauthlib.oauth2 import WebApplicationClient
 from requests_oauthlib import OAuth2Session
+from retrying import retry
+
+from integration_tests.errors import GqlError
 
 ENVNAME = os.getenv('ENVNAME', 'dev')
 
 
+def _retry_if_connection_error(exception):
+    """Return True if we should retry, False otherwise"""
+    return isinstance(exception, requests.exceptions.ConnectionError) or isinstance(exception, requests.ReadTimeout)
+
+
 class Client:
     def __init__(self, username, password):
         self.username = username
         self.password = password
         self.access_token = self._get_jwt_tokens()
 
-    @staticmethod
-    def _retry_if_connection_error(exception):
-        """Return True if we should retry, False otherwise"""
-        return isinstance(exception, requests.exceptions.ConnectionError) or isinstance(exception, requests.ReadTimeout)
-
     @retry(
         retry_on_exception=_retry_if_connection_error,
         stop_max_attempt_number=3,
         wait_random_min=1000,
         wait_random_max=3000,
     )
-    def query(self, query: str):
+    def query(self, query: dict):
         graphql_endpoint = os.path.join(os.environ['API_ENDPOINT'], 'graphql', 'api')
         headers = {'accesskeyid': 'none', 'SecretKey': 'none', 'Authorization': f'Bearer {self.access_token}'}
         r = requests.post(graphql_endpoint, json=query, headers=headers)
+        response = r.json()
+        if errors := response.get('errors'):
+            if any((response.get('data', {}) or {}).values()):  # check if there are data
+                logging.warning(f'{query=} returned both data and errors:\n {pformat(response)}')
+            else:
+                raise GqlError(errors)
         r.raise_for_status()
-        if errors := r.json().get('errors'):
-            raise GqlError(errors)
-
-        return DefaultMunch.fromDict(r.json())
+        return DefaultMunch.fromDict(response)
 
     def _get_jwt_tokens(self):
         token = uuid.uuid4()