Merge pull request #222 from eclecticiq/taxii2-implementation

Add taxii2 reference implementation

CHANGES.rst

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+0.3.0 (2022-04-13)
+------------------
+* Implement taxii2.1 support
+
 0.3.0a4 (2022-04-13)
 --------------------
 * Merge changes from 0.2.4 maintenance release

opentaxii/_version.py

@@ -3,4 +3,4 @@
 This module defines the package version for use in __init__.py and setup.py.
 """
 
-__version__ = '0.3.0a4'
+__version__ = '0.3.0'

opentaxii/auth/sqldb/api.py

@@ -47,15 +47,15 @@ def __init__(
 
     def authenticate(self, username, password):
         try:
-            account = Account.query.filter_by(username=username).one()
+            account = self.db.session.query(Account).filter_by(username=username).one()
         except exc.NoResultFound:
             return
         if not account.is_password_valid(password):
             return
         return self._generate_token(account.id, ttl=self.token_ttl_secs)
 
     def create_account(self, username, password, is_admin=False):
-        account = Account(username=username, is_admin=is_admin)
+        account = Account(username=username, is_admin=is_admin, permissions={})
         account.set_password(password)
         self.db.session.add(account)
         self.db.session.commit()
@@ -65,24 +65,24 @@ def get_account(self, token):
         account_id = self._get_account_id(token)
         if not account_id:
             return
-        account = Account.query.get(account_id)
+        account = self.db.session.query(Account).get(account_id)
         if not account:
             return
         return account_to_account_entity(account)
 
     def delete_account(self, username):
-        account = Account.query.filter_by(username=username).one_or_none()
+        account = self.db.session.query(Account).filter_by(username=username).one_or_none()
         if account:
             self.db.session.delete(account)
             self.db.session.commit()
 
     def get_accounts(self):
         return [
             account_to_account_entity(account)
-            for account in Account.query.all()]
+            for account in self.db.session.query(Account).all()]
 
     def update_account(self, obj, password=None):
-        account = Account.query.filter_by(username=obj.username).one_or_none()
+        account = self.db.session.query(Account).filter_by(username=obj.username).one_or_none()
         if not account:
             account = Account(username=obj.username)
             self.db.session.add(account)

opentaxii/cli/auth.py

@@ -1,6 +1,5 @@
-import sys
-
 import argparse
+import sys
 
 from opentaxii.cli import app
 
@@ -61,7 +60,7 @@ def update_account(argv=None):
                 return
             if args.field == 'admin':
                 account.is_admin = is_truely(args.value)
-                app.taxii_server.auth.update_account(account)
+                account = app.taxii_server.auth.update_account(account, None)
                 if account.is_admin:
                     print('now user is admin')
                 else:

opentaxii/cli/persistence.py

@@ -1,30 +1,34 @@
 import argparse
+
 import structlog
 import yaml
-
-from opentaxii.entities import Account
 from opentaxii.cli import app
+from opentaxii.entities import Account
 from opentaxii.local import context
 from opentaxii.utils import sync_conf_dict_into_db
 
 log = structlog.getLogger(__name__)
 
-local_admin = Account(
-    id=None, username="local-admin", permissions=None, is_admin=True)
+local_admin = Account(id=None, username="local-admin", permissions=None, is_admin=True)
 
 
 def sync_data_configuration():
     parser = argparse.ArgumentParser(
         description="Create services/collections/accounts",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument(
-        "config", help="YAML file with data configuration")
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("config", help="YAML file with data configuration")
     parser.add_argument(
-        "-f", "--force-delete", dest="force_deletion",
+        "-f",
+        "--force-delete",
+        dest="force_deletion",
         action="store_true",
-        help=("force deletion of collections and their content blocks "
-              "if collection is not defined in configuration file"),
-        required=False)
+        help=(
+            "force deletion of collections and their content blocks "
+            "if collection is not defined in configuration file"
+        ),
+        required=False,
+    )
     args = parser.parse_args()
     with open(args.config) as stream:
         config = yaml.safe_load(stream=stream)
@@ -33,41 +37,115 @@ def sync_data_configuration():
         # run as admin with full access
         context.account = local_admin
         sync_conf_dict_into_db(
-            app.taxii_server,
-            config,
-            force_collection_deletion=args.force_deletion)
+            app.taxii_server, config, force_collection_deletion=args.force_deletion
+        )
 
 
 def delete_content_blocks():
 
     parser = argparse.ArgumentParser(
         description=(
             "Delete content blocks from specified collections "
-            "with timestamp labels matching defined time window"),
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+            "with timestamp labels matching defined time window"
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
     parser.add_argument(
-        "-c", "--collection", action="append", dest="collection",
-        help="Collection to remove content blocks from", required=True)
+        "-c",
+        "--collection",
+        action="append",
+        dest="collection",
+        help="Collection to remove content blocks from",
+        required=True,
+    )
     parser.add_argument(
-        "-m", "--with-messages", dest="delete_inbox_messages",
+        "-m",
+        "--with-messages",
+        dest="delete_inbox_messages",
         action="store_true",
         help=("delete inbox messages associated with deleted content blocks"),
-        required=False)
+        required=False,
+    )
     parser.add_argument(
-        "--begin", dest="begin",
+        "--begin",
+        dest="begin",
         help="exclusive beginning of time window as ISO8601 formatted date",
-        required=True)
+        required=True,
+    )
     parser.add_argument(
-        "--end", dest="end",
-        help="inclusive ending of time window as ISO8601 formatted date")
+        "--end",
+        dest="end",
+        help="inclusive ending of time window as ISO8601 formatted date",
+    )
 
     args = parser.parse_args()
     with app.app_context():
         start_time = args.begin
         end_time = args.end
         for collection in args.collection:
-            app.taxii_server.persistence.delete_content_blocks(
+            app.taxii_server.servers.taxii1.persistence.delete_content_blocks(
                 collection,
                 with_messages=args.delete_inbox_messages,
                 start_time=start_time,
-                end_time=end_time)
+                end_time=end_time,
+            )
+
+
+def add_api_root():
+    """CLI command to add taxii2 api root to database."""
+    parser = argparse.ArgumentParser(
+        description=("Add a new taxii2 ApiRoot object."),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("-t", "--title", required=True, help="Title of the api root")
+    parser.add_argument(
+        "-d", "--description", required=False, help="Description of the api root"
+    )
+    parser.add_argument(
+        "--default", action="store_true", help="Set as default api root"
+    )
+
+    args = parser.parse_args()
+    with app.app_context():
+        app.taxii_server.servers.taxii2.persistence.api.add_api_root(
+            title=args.title, description=args.description, default=args.default
+        )
+
+
+def add_collection():
+    """CLI command to add taxii2 collection to database."""
+    existing_api_root_ids = [
+        str(api_root.id)
+        for api_root in app.taxii_server.servers.taxii2.persistence.api.get_api_roots()
+    ]
+    parser = argparse.ArgumentParser(
+        description=("Add a new taxii2 Collection object."),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-r",
+        "--rootid",
+        choices=existing_api_root_ids,
+        required=True,
+        help="Api root id of the collection",
+    )
+    parser.add_argument("-t", "--title", required=True, help="Title of the collection")
+    parser.add_argument(
+        "-d", "--description", required=False, help="Description of the collection"
+    )
+    parser.add_argument("-a", "--alias", required=False, help="alias of the collection")
+
+    args = parser.parse_args()
+    with app.app_context():
+        app.taxii_server.servers.taxii2.persistence.api.add_collection(
+            api_root_id=args.rootid,
+            title=args.title,
+            description=args.description,
+            alias=args.alias,
+        )
+
+
+def job_cleanup():
+    """CLI command to clean up taxii2 job logs that are >24h old."""
+    number_removed = app.taxii_server.servers.taxii2.persistence.api.job_cleanup()
+    print(f"{number_removed} removed")

opentaxii/common/entities.py

@@ -1,8 +1,29 @@
+def sorted_dicts(obj):
+    """
+    sort all dicts contained in obj, for repeatable repr
+    """
+    if isinstance(obj, dict):
+        response = {}
+        for key, value in sorted(obj.items()):
+            value = sorted_dicts(value)
+            response[key] = value
+    elif isinstance(obj, (list, tuple)):
+        response = type(obj)(sorted_dicts(item) for item in obj)
+    else:
+        response = obj
+    return response
+
 
 class Entity:
     '''Abstract TAXII entity class.
     '''
 
     def __repr__(self):
-        pairs = ["%s=%s" % (k, v) for k, v in sorted(self.__dict__.items())]
+        pairs = ["%s=%s" % (k, v) for k, v in sorted(sorted_dicts(self.__dict__).items())]
         return "%s(%s)" % (self.__class__.__name__, ", ".join(pairs))
+
+    def to_dict(self):
+        return {key: value for key, value in self.__dict__.items()}
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)

opentaxii/config.py

@@ -57,7 +57,12 @@ class ServerConfig(dict):
         "unauthorized_status",
         "hooks",
     )
-    VALID_TAXII2_OPTIONS = ("max_content_length",)
+    VALID_TAXII2_OPTIONS = (
+        "contact",
+        "description",
+        "max_content_length",
+        "title",
+    )
     ALL_VALID_OPTIONS = VALID_BASE_OPTIONS + VALID_TAXII_OPTIONS + VALID_TAXII1_OPTIONS
 
     def __init__(self, optional_env_var=CONFIG_ENV_VAR, extra_configs=None):
@@ -71,7 +76,7 @@ def __init__(self, optional_env_var=CONFIG_ENV_VAR, extra_configs=None):
         if env_var_path:
             configs.append(env_var_path)
         # 1. config built from env vars
-        configs.append(self._get_env_config())
+        configs.append(self._get_env_config(optional_env_var=optional_env_var))
 
         options = self._load_configs(*configs)
         options = self._clean_options(options)
@@ -85,11 +90,13 @@ def __init__(self, optional_env_var=CONFIG_ENV_VAR, extra_configs=None):
         super(ServerConfig, self).__init__(options)
 
     @staticmethod
-    def _get_env_config(env=os.environ):
+    def _get_env_config(env=os.environ, optional_env_var=None):
         result = _infinite_dict()
         for key, value in env.items():
             if not key.startswith(ENV_VAR_PREFIX):
                 continue
+            if key == optional_env_var:
+                continue
             key = key[len(ENV_VAR_PREFIX):].lstrip("_").lower()
             value = yaml.safe_load(value)
 

opentaxii/middleware.py

@@ -2,6 +2,9 @@
 
 import structlog
 from flask import Flask, request
+from marshmallow.exceptions import \
+    ValidationError as MarshmallowValidationError
+from werkzeug.exceptions import HTTPException
 
 from .exceptions import InvalidAuthHeader
 from .local import context, release_context
@@ -31,13 +34,15 @@ def create_app(server):
         "/<path:relative_path>",
         "opentaxii_services_view",
         server.handle_request,
-        methods=["POST", "OPTIONS"],
+        methods=["GET", "POST", "OPTIONS", "DELETE"],
     )
 
     app.register_blueprint(management, url_prefix="/management")
 
     app.register_error_handler(500, server.handle_internal_error)
     app.register_error_handler(StatusMessageException, server.handle_status_exception)
+    app.register_error_handler(HTTPException, server.handle_http_exception)
+    app.register_error_handler(MarshmallowValidationError, server.handle_validation_exception)
     app.before_request(functools.partial(create_context_before_request, server))
     app.after_request(cleanup_context)
     return app