2.6.2 Security features

.checkov.baseline

@@ -417,7 +417,7 @@
             ]
         },
         {
-            "file": "/cdk.out/asset.3045cb6b4340be1e173df6dcf6248d565aa849ceda3e2cf2c2f221ccee4bc1d6/pivotRole.yaml",
+            "file": "/cdk.out/asset.05d71d8b69cd4483d3c9db9120b556b718c72f349debbb79d461c74c4964b350/pivotRole.yaml",
             "findings": [
                 {
                     "resource": "AWS::IAM::ManagedPolicy.PivotRolePolicy0",
@@ -490,12 +490,6 @@
         {
             "file": "/checkov_environment_synth.json",
             "findings": [
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy19AC37181",
-                    "check_ids": [
-                        "CKV_AWS_111"
-                    ]
-                },
                 {
                     "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy2E85AF510",
                     "check_ids": [
@@ -508,24 +502,6 @@
                         "CKV_AWS_111"
                     ]
                 },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy5A19E75CA",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicyCC720210",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy1A0C96958",
-                    "check_ids": [
-                        "CKV_AWS_111"
-                    ]
-                },
                 {
                     "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy2B12D381A",
                     "check_ids": [
@@ -538,18 +514,6 @@
                         "CKV_AWS_111"
                     ]
                 },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy3E3CBA9E",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy56D7DC525",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
                 {
                     "resource": "AWS::Lambda::Function.CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536",
                     "check_ids": [
@@ -563,38 +527,34 @@
                     "resource": "AWS::Lambda::Function.GlueDatabaseLFCustomResourceHandler7FAF0F82",
                     "check_ids": [
                         "CKV_AWS_115",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.LakeformationDefaultSettingsHandler2CBEDB06",
                     "check_ids": [
                         "CKV_AWS_115",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.dataallGlueDbCustomResourceProviderframeworkonEventF8347BA7",
                     "check_ids": [
                         "CKV_AWS_115",
                         "CKV_AWS_116",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.dataallLakeformationDefaultSettingsProviderframeworkonEventBB660E32",
                     "check_ids": [
                         "CKV_AWS_115",
                         "CKV_AWS_116",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
-                    "resource": "AWS::S3::Bucket.EnvironmentDefaultBucket78C3A8B0",
+                    "resource": "AWS::S3::Bucket.EnvironmentDefaultLogBucket7F0EFAB3",
                     "check_ids": [
                         "CKV_AWS_18"
                     ]
@@ -653,6 +613,25 @@
                 }
             ]
         },
+        {
+            "file": "/checkov_pipeline_synth.json",
+            "findings": [
+                {
+                    "resource": "AWS::IAM::Role.PipelineRoleDCFDBB91",
+                    "check_ids": [
+                        "CKV_AWS_107",
+                        "CKV_AWS_108",
+                        "CKV_AWS_111"
+                    ]
+                },
+                {
+                    "resource": "AWS::S3::Bucket.thistableartifactsbucketDB1C8C64",
+                    "check_ids": [
+                        "CKV_AWS_18"
+                    ]
+                }
+            ]
+        },
         {
             "file": "/frontend/docker/prod/Dockerfile",
             "findings": [

.github/workflows/snyk.yaml

@@ -0,0 +1,31 @@
+name: Snyk
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    - cron: "0 9 * * 1"  # runs each Monday at 9:00 UTC
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  security:
+    strategy:
+      matrix:
+        python-version: [3.9]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snyk/actions/setup@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install All Requirements
+        run: make install
+      - name: Run Snyk to check for vulnerabilities
+        run: snyk test --all-projects --detection-depth=5 --severity-threshold=high
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

Makefile

@@ -16,7 +16,7 @@ venv:
 	@python3 -m venv "venv"
 	@/bin/bash -c "source venv/bin/activate"
 
-install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests
+install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests install-integration-tests install-custom-auth install-userguide
 
 upgrade-pip:
 	pip install --upgrade pip setuptools
@@ -36,6 +36,12 @@ install-tests:
 install-integration-tests:
 	pip install -r tests_new/integration_tests/requirements.txt
 
+install-custom-auth:
+	pip install -r deploy/custom_resources/custom_authorizer/requirements.txt
+
+install-userguide:
+	pip install -r documentation/userguide/requirements.txt
+
 lint:
 	pip install ruff
 	ruff check --fix

backend/api_handler.py

@@ -23,6 +23,7 @@
 from dataall.base.db import get_engine
 from dataall.base.loader import load_modules, ImportMode
 
+from graphql.pyutils import did_you_mean
 
 logger = logging.getLogger()
 logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
@@ -32,6 +33,11 @@
 for name in ['boto3', 's3transfer', 'botocore', 'boto']:
     logging.getLogger(name).setLevel(logging.ERROR)
 
+ALLOW_INTROSPECTION = True if os.getenv('ALLOW_INTROSPECTION') == 'True' else False
+
+if not ALLOW_INTROSPECTION:
+    did_you_mean.__globals__['MAX_LENGTH'] = 0
+
 load_modules(modes={ImportMode.API})
 SCHEMA = bootstrap_schema()
 TYPE_DEFS = gql(SCHEMA.gql(with_directives=False))
@@ -137,7 +143,9 @@ def handler(event, context):
     else:
         raise Exception(f'Could not initialize user context from event {event}')
 
-    success, response = graphql_sync(schema=executable_schema, data=query, context_value=app_context)
+    success, response = graphql_sync(
+        schema=executable_schema, data=query, context_value=app_context, introspection=ALLOW_INTROSPECTION
+    )
 
     dispose_context()
     response = json.dumps(response)

backend/dataall/__init__.py

@@ -1,2 +1,13 @@
 from . import core, version
 from .base import utils, db, api
+import logging
+import os
+import sys
+
+logging.basicConfig(
+    level=os.environ.get('LOG_LEVEL', 'INFO'),
+    handlers=[logging.StreamHandler(sys.stdout)],
+    format='[%(levelname)s] %(message)s',
+)
+for name in ['boto3', 's3transfer', 'botocore', 'boto', 'urllib3']:
+    logging.getLogger(name).setLevel(logging.ERROR)

backend/dataall/base/cdkproxy/requirements.txt

@@ -1,17 +1,12 @@
-aws-cdk-lib==2.99.0
-boto3==1.28.23
-boto3-stubs==1.28.23
-botocore==1.31.23
+aws-cdk-lib==2.160.0
+boto3==1.35.26
+boto3-stubs==1.35.26
 cdk-nag==2.7.2
-constructs==10.0.73
-starlette==0.36.3
-fastapi == 0.109.2
-Flask==2.3.2
+fastapi == 0.115.5
 PyYAML==6.0
 requests==2.32.2
 tabulate==0.8.9
 uvicorn==0.15.0
-werkzeug==3.0.3
-constructs>=10.0.0,<11.0.0
+werkzeug==3.0.6
 git-remote-codecommit==1.16
 aws-ddk-core==1.3.0

backend/dataall/base/context.py

@@ -4,7 +4,7 @@
 that in the request scope
 
 The class uses Flask's approach to handle request: ThreadLocal
-That approach should work fine for AWS Lambdas and local server that uses Flask app
+That approach should work fine for AWS Lambdas and local server that uses FastApi app
 """
 
 from dataclasses import dataclass

backend/dataall/base/feature_toggle_checker.py

@@ -2,6 +2,8 @@
 Contains decorators that check if a feature has been enabled or not
 """
 
+import functools
+
 from dataall.base.config import config
 from dataall.base.utils.decorator_utls import process_func
 
@@ -10,6 +12,7 @@ def is_feature_enabled(config_property: str):
     def decorator(f):
         fn, fn_decorator = process_func(f)
 
+        @functools.wraps(fn)
         def decorated(*args, **kwargs):
             value = config.get_property(config_property)
             if not value:

backend/dataall/base/utils/naming_convention.py

@@ -1,28 +1,46 @@
 from enum import Enum
-
+import re
 from .slugify import slugify
 
 
 class NamingConventionPattern(Enum):
-    S3 = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
+    S3 = {
+        'regex': '[^a-zA-Z0-9-]',
+        'separator': '-',
+        'max_length': 63,
+        'valid_external_regex': '(?!(^xn--|.+-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$',
+    }
+    KMS = {'regex': '[^a-zA-Z0-9-]$', 'separator': '-', 'max_length': 63, 'valid_external_regex': '^[a-zA-Z0-9_-]+$'}
     IAM = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 63}  # Role names up to 64 chars
     IAM_POLICY = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 128}  # Policy names up to 128 chars
-    GLUE = {'regex': '[^a-zA-Z0-9_]', 'separator': '_', 'max_length': 240}  # Limit 255 - 15 extra chars buffer
+    GLUE = {
+        'regex': '[^a-zA-Z0-9_]',
+        'separator': '_',
+        'max_length': 240,
+        'valid_external_regex': '^[a-zA-Z0-9_]+$',
+    }  # Limit 255 - 15 extra chars buffer
     GLUE_ETL = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 52}
     NOTEBOOK = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
     MLSTUDIO_DOMAIN = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
     DEFAULT = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 63}
+    DEFAULT_SEARCH = {'regex': '[^a-zA-Z0-9-_:. ]'}
     OPENSEARCH = {'regex': '[^a-z0-9-]', 'separator': '-', 'max_length': 27}
     OPENSEARCH_SERVERLESS = {'regex': '[^a-z0-9-]', 'separator': '-', 'max_length': 31}
+    DATA_FILTERS = {'regex': '[^a-z0-9_]', 'separator': '_', 'max_length': 31}
+    REDSHIFT_DATASHARE = {
+        'regex': '[^a-zA-Z0-9_]',
+        'separator': '_',
+        'max_length': 1000,
+    }  # Maximum length of 2147483647
 
 
 class NamingConventionService:
     def __init__(
         self,
         target_label: str,
-        target_uri: str,
         pattern: NamingConventionPattern,
-        resource_prefix: str,
+        target_uri: str = '',
+        resource_prefix: str = '',
     ):
         self.target_label = target_label
         self.target_uri = target_uri if target_uri else ''
@@ -37,4 +55,8 @@ def build_compliant_name(self) -> str:
         separator = NamingConventionPattern[self.service].value['separator']
         max_length = NamingConventionPattern[self.service].value['max_length']
         suffix = f'-{self.target_uri}' if len(self.target_uri) else ''
-        return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length- len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
+        return f"{slugify(self.resource_prefix + '-' + self.target_label[:(max_length - len(self.resource_prefix + self.target_uri))] + suffix, regex_pattern=fr'{regex}', separator=separator, lowercase=True)}"
+
+    def sanitize(self):
+        regex = NamingConventionPattern[self.service].value['regex']
+        return re.sub(regex, '', self.target_label)

backend/dataall/core/environment/api/queries.py

@@ -32,6 +32,7 @@
 
 getTrustAccount = gql.QueryField(
     name='getTrustAccount',
+    args=[gql.Argument(name='organizationUri', type=gql.NonNullableType(gql.String))],
     type=gql.String,
     resolver=get_trust_account,
     test_scope='Environment',