From e3b6f5f3984d4ef9178c4046d6bd4b381e024a80 Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Fri, 25 Feb 2022 18:26:35 +1100 Subject: [PATCH 01/10] Add artifactory_catalog_connector --- component-catalog-connectors/README.md | 1 + .../artifactory-connector/.flake8 | 30 ++ .../artifactory-connector/MANIFEST.in | 38 +++ .../artifactory-connector/Makefile | 54 ++++ .../artifactory-connector/README.md | 203 ++++++++++++ .../artifactory_catalog_connector/__init__.py | 15 + .../artifactory_catalog_connector/_version.py | 16 + .../artifactory-catalog.json | 170 ++++++++++ .../artifactory_catalog_connector.py | 293 ++++++++++++++++++ .../artifactory_schema_provider.py | 54 ++++ .../packaging_ports.py | 92 ++++++ .../artifactory-connector/setup.cfg | 53 ++++ .../artifactory-connector/setup.py | 69 +++++ .../test_requirements.txt | 1 + .../connector-directory.md | 5 +- 15 files changed, 1092 insertions(+), 2 deletions(-) create mode 100644 component-catalog-connectors/artifactory-connector/.flake8 create mode 100644 component-catalog-connectors/artifactory-connector/MANIFEST.in create mode 100644 component-catalog-connectors/artifactory-connector/Makefile create mode 100644 component-catalog-connectors/artifactory-connector/README.md create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/__init__.py create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/_version.py create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_schema_provider.py create mode 100644 component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/packaging_ports.py create mode 100644 component-catalog-connectors/artifactory-connector/setup.cfg create mode 100644 component-catalog-connectors/artifactory-connector/setup.py create mode 100644 component-catalog-connectors/artifactory-connector/test_requirements.txt diff --git a/component-catalog-connectors/README.md b/component-catalog-connectors/README.md index b778606..7617e86 100644 --- a/component-catalog-connectors/README.md +++ b/component-catalog-connectors/README.md @@ -7,6 +7,7 @@ Component catalog connectors provide Elyra's Visual Pipeline Editor access to [l This directory contains component catalog connector implementations for - [Kubeflow Pipelines example components](kfp-example-components-connector) - [Apache Airflow example operators](airflow-example-components-connector) +- [Artifactory](artifactory-connector) - [Machine Learning Exchange](mlx-connector) The connectors listed above are maintained by the Elyra community. You can find a complete list of available connectors on [this page](connector-directory.md) or learn how to [build your own](build-a-custom-connector.md). diff --git a/component-catalog-connectors/artifactory-connector/.flake8 b/component-catalog-connectors/artifactory-connector/.flake8 new file mode 100644 index 0000000..05716f2 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/.flake8 @@ -0,0 +1,30 @@ +[flake8] +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +exclude = + __init__.py +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # Allow breaks after binary operators + W504, + # Include name with TODOs as in # TODO(yourname) + H101, + # Do not import more than one module per line + H301, + # Alphabetically order imports by the full module path + H306, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405 +max-line-length = 120 + diff --git a/component-catalog-connectors/artifactory-connector/MANIFEST.in b/component-catalog-connectors/artifactory-connector/MANIFEST.in new file mode 100644 index 0000000..64a3556 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/MANIFEST.in @@ -0,0 +1,38 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# exclude from ANY directory +global-exclude *.ipynb +global-exclude *.py[cod] +global-exclude __pycache__ +global-exclude .git +global-exclude .ipynb_checkpoints +global-exclude .DS_Store +global-exclude *.sh +global-exclude docs +global-exclude tests + +# explicit includes +include CONTRIBUTING.md +include README.md +include LICENSE +include dist/*.tgz + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +# include the connector schema file +include artifactory_catalog_connector/artifactory-catalog.json diff --git a/component-catalog-connectors/artifactory-connector/Makefile b/component-catalog-connectors/artifactory-connector/Makefile new file mode 100644 index 0000000..d98abb4 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/Makefile @@ -0,0 +1,54 @@ +# +# Copyright 2021-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: help clean lint test-dependencies source-install install dist +.PHONY: publish test-publish + +SHELL:=/bin/bash + +PACKAGE_NAME=elyra-artifactory-catalog-connector +PACKAGE_PATH=artifactory_catalog_connector + +help: +# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +clean: + rm -rf dist/ + rm -rf build/ + rm -rf *.egg-info + - pip uninstall -y $(PACKAGE_NAME) + +test-dependencies: + @pip install -q -r test_requirements.txt + +lint: test-dependencies + flake8 $(PACKAGE_PATH) + +dist: clean lint ## Build distribution + python setup.py bdist_wheel sdist + +source-install: dist ## Install component connector package from source + pip install . + +install: ## Install component connector package from PyPI + pip install $(PACKAGE_NAME) + +test-publish: dist ## Upload package to test PyPI + twine upload --repository testpypi dist/* + +publish: dist ## Upload package to PyPI + twine upload dist/* diff --git a/component-catalog-connectors/artifactory-connector/README.md b/component-catalog-connectors/artifactory-connector/README.md new file mode 100644 index 0000000..537d0e1 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/README.md @@ -0,0 +1,203 @@ +## Artifactory component catalog connector + +This catalog connector enables Elyra to load pipelines components from a generic-type Artifactory repo. + +## Install the connector + +You can install the Artifactory catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. + +### Prerequisites + +- [Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above) +- A [generic-type Artifactory repo](https://www.jfrog.com/confluence/display/JFROG/Repository+Management#RepositoryManagement-GenericRepositories) + +### Install from PyPI + + ``` + $ pip install elyra-artifactory-catalog-connector + ``` + +### Install from source code + + ``` + $ git clone https://github.com/elyra-ai/examples.git + $ cd examples/component-catalog-connectors/artifactory-connector/ + $ make source-install + ``` + +## Use the connector + +### Add a new Artifactory catalog + +1. Launch JupyterLab +2. Open the [`Manage Components` panel](https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui) +3. Click `+` then `New Artifactory Component Catalog` +4. Specify a "name" for the catalog +5. (Optional) Specify a "category name" under which the catalog's components will be organized in the palette +6. Specify the required "Source" parameters: + - `artifactory_url` + - `repository_name` + - `repository_path` + - `max_recursion_depth` + - `max_files_per_folder` + - `file_filter` + - `file_ordering` +7. (Optional) Specify the optional "Source" parameters: + - `artifactory_username` + - `artifactory_password` +8. Save the catalog entry +9. Open the Visual Pipeline Editor and expand the palette. The catalog components are displayed. + +### Recommended Usage + +We recommend that you version your component YAML rather than having users always point to the latest version of each component. +Otherwise, when changes are made to `inputs`/`outputs`/`implementation`, existing pipelines may break or do unexpected things. + +> 🟦 __Tip__ 🟦 +> +> Include the version in each component's `name` so they can be distinguished in the Elyra UI. +> +> For example: +> ```yaml +> name: My Component - 1.0.0 +> description: "" +> inputs: [] +> outputs: [] +> implementation: {} +> ``` + +Elyra only stores pointers to the source-catalog of each component in its `.pipeline` files. +This means your catalogs must continue to include ALL old component versions, otherwise old pipelines will be unable to run. + +You can solve this problem with the `Artifactory Catalog Connector` by adding TWO catalog instances for your single Artifactory repo: + +1. that has only the latest version of each component: + - `max_files_per_folder = 1` + - `file_ordering = VERSION_DESCENDING` +2. that has all versions of each component: + - `max_files_per_folder = -1` + +> 🟦 __Tip__ 🟦 +> +> Name your component YAML files with consistent prefixes before the version. +> +> This is because `VERSION_DESCENDING` treats the whole file-name as a version. +> For example, `aaaa-1.0.0.yaml` would be sorted before `bbbb-9.0.0.yaml`. +> +> Alternatively, don't include any prefix on file names, for example `1.0.0.yaml`, `1.1.0.yaml`. + + +### Example Configs + +Assume an artifactory server `https://example.com/artifactory/` has a repository called `elyra-components` +with the following folder structure: + +``` +component_1/ + __COMPONENT__ + component-1.0.9.yaml + component-1.0.10.yaml +component_2/ + hidden_component/ + __COMPONENT__ + component-1.0.0.yaml + component-1.1.0.yaml + __COMPONENT__ + component-1.0.0.yaml + component-1.1.0.yaml +component_3/ + component-1.0.0.yaml + component-1.1.0.yaml +``` + +__Example 1:__ + +``` +Configs: +========= +artifactory_url = https://example.com/artifactory/ +repository_name = elyra-components +repository_path = / +max_recursion_depth = 3 +max_files_per_folder = -1 +file_filter = *.yaml +file_ordering = VERSION_DESCENDING + +Matched: +========= +https://example.com/artifactory/elyra-components/component_1/component-1.0.9.yaml +https://example.com/artifactory/elyra-components/component_1/component-1.0.10.yaml +https://example.com/artifactory/elyra-components/component_2/component-1.0.0.yaml +https://example.com/artifactory/elyra-components/component_2/component-1.1.0.yaml + +Notes: +========= +- the `component_3/` files are not matched as this folder does not contain a `__COMPONENT__` marker +- the `component_2/hidden_component/` files are not matched as recursion stops at the first `__COMPONENT__` marker +``` + +__Example 2:__ + +``` +Configs: +========= +artifactory_url = https://example.com/artifactory/ +repository_name = elyra-components +repository_path = / +max_recursion_depth = 0 +max_files_per_folder = -1 +file_filter = *.yaml +file_ordering = VERSION_DESCENDING + +Matched: +========= +N/A + +Notes: +========= +- no files are matched, as `max_recursion_depth` is `0` +``` + +__Example 3:__ + +``` +Configs: +========= +artifactory_url = https://example.com/artifactory/ +repository_name = elyra-components +repository_path = / +max_recursion_depth = 3 +max_files_per_folder = 1 +file_filter = *.yaml +file_ordering = VERSION_DESCENDING + +Matched: +========= +https://example.com/artifactory/elyra-components/component_1/component-1.0.10.yaml +https://example.com/artifactory/elyra-components/component_2/component-1.1.0.yaml + +Notes: +========= +- the `file_ordering` is applied separately within each folder +- as `max_files_per_folder` is `1`, only ONE file from each folder is matched +- as `file_ordering` is `VERSION_DESCENDING`, the file names are ordered as if they are version numbers + (we use `packaging.version.LegacyVersion()` to preform the sort) +- the whole file-name is treated as a version, so "aaaa-1.0.0.yaml" is sorted before "bbbb-9.0.0.yaml" + (take care not to change your file-name prefixes, or alternatively don't include a prefix and use "1.0.0.ymal") +``` + +## Uninstall the connector + +1. Remove all Artifactory catalog entries from the `Manage Components` panel +2. Stop JupyterLab +3. Uninstall the `elyra-artifactory-catalog-connector` package: + ``` + $ pip uninstall elyra-artifactory-catalog-connector + ``` + +## Troubleshooting + +- __Problem:__ The palette does not display any components from the configured catalog. + - __Solution:__ If the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. +- __Problem:__ The pallet does not reflect the current state of the Artifactory repo. + - __Solution:__ Trigger the catalog to refresh by editing the catalog, and clicking "SAVE & CLOSE" (without making any changes). diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/__init__.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/__init__.py new file mode 100644 index 0000000..e443872 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/_version.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/_version.py new file mode 100644 index 0000000..cbfc482 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/_version.py @@ -0,0 +1,16 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.1.0" diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json new file mode 100644 index 0000000..838bc9f --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json @@ -0,0 +1,170 @@ +{ + "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json", + "title": "Artifactory Component Catalog", + "name": "artifactory-catalog", + "schemaspace": "component-catalogs", + "schemaspace_id": "8dc89ca3-4b90-41fd-adb9-9510ad346620", + "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", + "uihints": { + "title": "Artifactory Component Catalog", + "icon": "", + "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/artifactory-connector" + }, + "properties": { + "schema_name": { + "title": "Schema Name", + "description": "The schema associated with this instance", + "type": "string", + "const": "artifactory-catalog" + }, + "display_name": { + "title": "Display Name", + "description": "Display name of this Component Catalog", + "type": "string", + "minLength": 1 + }, + "metadata": { + "description": "Additional data specific to this metadata", + "type": "object", + "properties": { + "description": { + "title": "Description", + "description": "Description of this Component Catalog", + "type": "string", + "default": "Artifactory component catalog" + }, + "runtime_type": { + "title": "Runtime", + "description": "List of runtime types this catalog supports", + "type": "string", + "enum": [ + "KUBEFLOW_PIPELINES" + ], + "default": "KUBEFLOW_PIPELINES", + "uihints": { + "field_type": "dropdown" + } + }, + "categories": { + "title": "Category Names", + "description": "Assign the components in the catalog to one or more categories, to group them in the visual pipeline editor palette.", + "type": "array", + "items": { + "type": "string", + "maxLength": 18 + }, + "uihints": { + "field_type": "array", + "category": "Component Categories" + } + }, + "artifactory_url": { + "title": "Artifactory URL", + "description": "URL of the Artifactory server", + "type": "string", + "format": "uri", + "uihints": { + "category": "Source", + "placeholder": "https://example.org/artifactory/" + } + }, + "artifactory_username": { + "title": "Artifactory Username", + "description": "Username for the Artifactory server", + "type": "string", + "uihints": { + "category": "Source" + } + }, + "artifactory_password": { + "title": "Artifactory Password/API-Key", + "description": "Password or API Key for the Artifactory server", + "type": "string", + "uihints": { + "secure": true, + "category": "Source" + } + }, + "repository_name": { + "title": "Repository Name", + "description": "Name of the Artifactory repository", + "type": "string", + "pattern": "^[^\\/\\:|?*\"'<>+]+$", + "uihints": { + "category": "Source" + } + }, + "repository_path": { + "title": "Repository Path", + "description": "Path of folder in repository to search under", + "type": "string", + "default": "/", + "uihints": { + "category": "Source" + } + }, + "max_recursion_depth": { + "title": "Maximum Recursion Depth", + "description": "Maximum folder depth to recurse looking for '__COMPONENT__' files", + "type": "string", + "pattern": "^[0-9]+$", + "default": "0", + "uihints": { + "category": "Source" + } + }, + "max_files_per_folder": { + "title": "Maximum Files Per Folder", + "description": "Maximum number of files returned from each folder. ('-1' is unlimited)", + "type": "string", + "pattern": "^-1|[0-9]+$", + "default": "-1", + "uihints": { + "category": "Source" + } + }, + "file_filter": { + "title": "File Filter", + "description": "Unix-like file name filter. ('*' match everything; '?' any single character; '[seq]' character in seq; '[!seq]' character not in seq, '[0-9]' any number)", + "default": "*.yaml", + "type": "string", + "uihints": { + "category": "Source" + } + }, + "file_ordering": { + "title": "File Ordering", + "description": "Order in which files are processed per folder", + "type": "string", + "enum": [ + "NAME_ASCENDING", + "NAME_DESCENDING", + "VERSION_ASCENDING", + "VERSION_DESCENDING" + ], + "default": "VERSION_DESCENDING", + "uihints": { + "field_type": "dropdown", + "category": "Source" + } + } + }, + "required": [ + "runtime_type", + "artifactory_url", + "repository_name", + "repository_path", + "max_recursion_depth", + "max_files_per_folder", + "file_filter", + "file_ordering" + ] + } + }, + "required": [ + "schema_name", + "display_name", + "metadata" + ] +} diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py new file mode 100644 index 0000000..0180fed --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py @@ -0,0 +1,293 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import fnmatch +from http import HTTPStatus +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from urllib.parse import urlparse + +import requests +from artifactory_catalog_connector import packaging_ports +from elyra.pipeline.catalog_connector import ComponentCatalogConnector +from requests.auth import HTTPBasicAuth, AuthBase + + +class ArtifactoryApiKeyAuth(AuthBase): + """ + A `requests.auth.AuthBase` implementation that sets the "X-JFrog-Art-Api" header. + """ + + def __init__(self, api_key): + self.api_key = api_key + + def __call__(self, r): + r.headers["X-JFrog-Art-Api"] = self.api_key + return r + + +def get_folder_info( + api_base_url: str, + api_auth: Optional[AuthBase], + repository_name: str, + folder_path: str, + file_filter: str, +) -> (List[str], List[str], bool): + """ + Run a "folder info" query against an Artifactory repo to get lists of child files/folders. + + https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-FolderInfo + + :param api_base_url: the base url of the artifactory server + :param api_auth: an instance of requests.auth.AuthBase + :param repository_name: the name of the artifactory repository + :param folder_path: the path of the folder which the query will be run against + :param file_filter: an `fnmatch.fnmatch()` pattern to filter the returned child files + :return: (child_folders, child_files, is_component_folder) + """ + req_url = f"{api_base_url}/api/storage/{repository_name}/{folder_path.strip('/')}" + resp = requests.get(req_url, auth=api_auth) + + if resp.status_code != HTTPStatus.OK: + raise RuntimeError( + f"Failed to get FolderInfo from '{req_url}'... " + f"Got unhandled HTTP Status {resp.status_code}, expected 200... " + f"{resp.text}" + ) + expected_content_type = ( + "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json" + ) + if resp.headers["Content-Type"] != expected_content_type: + raise RuntimeError( + f"Failed to get FolderInfo from '{req_url}'... " + f"Got unexpected Content-Type '{resp.headers['Content-Type']}', " + f"expected '{expected_content_type}'" + ) + + child_folders = [] + child_files = [] + is_component_folder = False + + for child in resp.json().get("children", []): + if child["folder"]: + folder_path = child["uri"] + child_folders.append(folder_path) + else: + file_path = child["uri"].lstrip("/") + if file_path == "__COMPONENT__": + is_component_folder = True + elif fnmatch.fnmatch(file_path, file_filter): + child_files.append(file_path) + + return child_folders, child_files, is_component_folder + + +def recursively_get_components( + api_base_url: str, + api_auth: Optional[AuthBase], + repository_name: str, + file_filter: str, + file_ordering: str, + max_recursion_depth: int, + max_files_per_folder: int, + current_folder_path: str, + current_recursion_depth: int, +) -> List[Dict[str, Any]]: + """ + A function which recursively traverses an Artifactory repo to return elyra `component_metadata` instances. + + NOTE: only returns `component_metadata` instances from folders which contain a "__COMPONENT__" marker file. + + :param api_base_url: the base url of the artifactory server + :param api_auth: an instance of requests.auth.AuthBase + :param repository_name: the name of the artifactory repository + :param file_filter: an `fnmatch.fnmatch()` pattern to filter the returned files + :param file_ordering: how the files in each folder are ordered (used in conjunction with `max_files_per_folder`) + :param max_recursion_depth: max folder depth to recurse looking for "__COMPONENT__" files + :param max_files_per_folder: max number of files returned from each folder (-1 is unlimited) + :param current_folder_path: the path of the current folder that is being traversed + :param current_recursion_depth: the current folder depth + :return: a list of elyra `component_metadata` instances + """ + component_list = [] + + if current_recursion_depth > max_recursion_depth: + return component_list + + child_folders, child_files, is_component_folder = get_folder_info( + api_base_url=api_base_url, + api_auth=api_auth, + repository_name=repository_name, + folder_path=current_folder_path, + file_filter=file_filter, + ) + + if is_component_folder: + if max_files_per_folder >= 0: + if file_ordering == "NAME_ASCENDING": + child_files.sort() + elif file_ordering == "NAME_DESCENDING": + child_files.sort(reverse=True) + elif file_ordering == "VERSION_ASCENDING": + child_files.sort(key=packaging_ports.legacy_version) + elif file_ordering == "VERSION_DESCENDING": + child_files.sort(key=packaging_ports.legacy_version, reverse=True) + + child_files = child_files[0:min(max_files_per_folder, len(child_files))] + + for relative_path in child_files: + absolute_file_path = ( + f"{current_folder_path.strip('/')}/{relative_path.strip('/')}" + ) + component_list.append( + {"url": f"{api_base_url}/{repository_name}/{absolute_file_path}"} + ) + else: + for relative_path in child_folders: + absolute_folder_path = ( + f"{current_folder_path.strip('/')}/{relative_path.strip('/')}" + ) + component_list += recursively_get_components( + api_base_url=api_base_url, + api_auth=api_auth, + repository_name=repository_name, + file_filter=file_filter, + file_ordering=file_ordering, + max_recursion_depth=max_recursion_depth, + max_files_per_folder=max_files_per_folder, + current_folder_path=absolute_folder_path, + current_recursion_depth=current_recursion_depth + 1, + ) + + return component_list + + +class ArtifactoryComponentCatalogConnector(ComponentCatalogConnector): + """ + Read component definitions from an Artifactory catalog + """ + + def get_catalog_entries( + self, catalog_metadata: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """ + Returns a list of component_metadata instances, one per component found in the given registry. + The form that component_metadata takes is determined by requirements of the reader class. + + :param catalog_metadata: the dictionary-form of the Metadata instance for a single registry + """ + component_list = [] + + artifactory_url = catalog_metadata["artifactory_url"] + artifactory_username = catalog_metadata.get("artifactory_username") + artifactory_password = catalog_metadata.get("artifactory_password") + repository_name = catalog_metadata["repository_name"] + repository_path = catalog_metadata["repository_path"] + max_recursion_depth = int(catalog_metadata["max_recursion_depth"]) + max_files_per_folder = int(catalog_metadata["max_files_per_folder"]) + file_filter = catalog_metadata["file_filter"] + file_ordering = catalog_metadata["file_ordering"] + + url_obj = urlparse(artifactory_url) + api_base_url = f"{url_obj.scheme}://{url_obj.netloc}/{url_obj.path.strip('/')}" + + api_auth = None + if artifactory_username and artifactory_password: + api_auth = HTTPBasicAuth( + username=artifactory_username, password=artifactory_password + ) + elif artifactory_password: + api_auth = ArtifactoryApiKeyAuth(api_key=artifactory_password) + + try: + self.log.debug("Retrieving component list from Artifactory catalog.") + + component_list += recursively_get_components( + api_base_url=api_base_url, + api_auth=api_auth, + repository_name=repository_name, + file_filter=file_filter, + file_ordering=file_ordering, + max_recursion_depth=max_recursion_depth, + max_files_per_folder=max_files_per_folder, + current_folder_path=repository_path, + current_recursion_depth=0, + ) + + except Exception as ex: + self.log.error( + f"Error retrieving component list from Artifactory catalog: {ex}" + ) + + return component_list + + def read_catalog_entry( + self, catalog_entry_data: Dict[str, Any], catalog_metadata: Dict[str, Any] + ) -> Optional[str]: + """ + Fetch the component that is identified by catalog_entry_data from + the Artifactory catalog. + + :param catalog_entry_data: a dictionary that contains the information needed to read the content + of the component definition + :param catalog_metadata: the metadata associated with the catalog in which this catalog entry is + stored; in addition to catalog_entry_data, catalog_metadata may also be + needed to read the component definition for certain types of catalogs + + :returns: the content of the given catalog entry's definition in string form + """ + artifactory_username = catalog_metadata.get("artifactory_username") + artifactory_password = catalog_metadata.get("artifactory_password") + + url = catalog_entry_data.get("url") + if url is None: + self.log.error("Artifactory component source must contain `url`.") + return None + + api_auth = None + if artifactory_username and artifactory_password: + api_auth = HTTPBasicAuth( + username=artifactory_username, password=artifactory_password + ) + elif artifactory_password: + api_auth = ArtifactoryApiKeyAuth(api_key=artifactory_password) + + resp = requests.get(url, auth=api_auth) + if resp.status_code != HTTPStatus.OK: + self.log.error( + f"Failed to read component from '{url}'... " + f"Got unhandled HTTP Status {resp.status_code}, expected 200" + ) + return None + if resp.headers["Content-Type"] != "text/plain": + self.log.error( + f"Failed to read component from '{url}'... " + f"Got unexpected Content-Type '{resp.headers['Content-Type']}', expected 'text/plain'" + ) + return None + + return resp.text + + def get_hash_keys(self) -> List[Any]: + """ + Identifies the unique Artifactory catalog key that method read_catalog_entry + can use to fetch a component from the catalog. Method get_catalog_entries + retrieves the list of available key values from the catalog. + + :returns: a list of keys + """ + return ["url"] diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_schema_provider.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_schema_provider.py new file mode 100644 index 0000000..f472aea --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_schema_provider.py @@ -0,0 +1,54 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging +from pathlib import Path +from typing import Dict +from typing import List + +from elyra.metadata.schema import SchemasProvider + + +class ArtifactorySchemasProvider(SchemasProvider): + """ + Enables catalog connector for Artifactory + """ + + def get_schemas(self) -> List[Dict]: + """ + Return the Artifactory catalog connector schema + """ + # use Elyra logger + log = logging.getLogger("ElyraApp") + catalog_schema_defs = [] + try: + # load Artifactory catalog schema definition + catalog_connector_schema_file = ( + Path(__file__).parent / "artifactory-catalog.json" + ) + log.debug( + f"Reading Artifactory catalog connector schema from {catalog_connector_schema_file}" + ) + with open(catalog_connector_schema_file, "r") as fp: + catalog_connector_schema = json.load(fp) + catalog_schema_defs.append(catalog_connector_schema) + except Exception as ex: + log.error( + "Error reading Artifactory catalog connector " + f"schema {catalog_connector_schema_file}: {ex}" + ) + + return catalog_schema_defs diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/packaging_ports.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/packaging_ports.py new file mode 100644 index 0000000..163d68f --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/packaging_ports.py @@ -0,0 +1,92 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import re +from typing import Iterator, Tuple, List + +""" +Provides a port of `packaging.version.LegacyVersion()` which will be removed in a future `packaging` release. +Used in our implementation of "VERSION_ASCENDING" and "VERSION_DESCENDING" for `file_ordering`. +""" + +# source: https://github.com/pypa/packaging/blob/21.3/packaging/version.py#L32 +LegacyCmpKey = Tuple[int, Tuple[str, ...]] + +# source: https://github.com/pypa/packaging/blob/21.3/packaging/version.py#L168 +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) + +# source: https://github.com/pypa/packaging/blob/21.3/packaging/version.py#L170-L176 +_legacy_version_replacement_map = { + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", +} + + +# source: https://github.com/pypa/packaging/blob/21.3/packaging/version.py#L179-L193 +def _parse_version_parts(s: str) -> Iterator[str]: + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +# source: https://github.com/pypa/packaging/blob/21.3/packaging/version.py#L196-L220 +def _legacy_cmpkey(version: str) -> LegacyCmpKey: + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts: List[str] = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + + return epoch, tuple(parts) + + +def legacy_version(version: str) -> LegacyCmpKey: + """ + Provides the same behaviour as `packaging.version.LegacyVersion()` when used as a sort-key function. + + :param version: the version string to extract a sort key from + :return: a sort key + """ + return _legacy_cmpkey(version) diff --git a/component-catalog-connectors/artifactory-connector/setup.cfg b/component-catalog-connectors/artifactory-connector/setup.cfg new file mode 100644 index 0000000..99fa0cb --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/setup.cfg @@ -0,0 +1,53 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[bdist_wheel] +universal=0 + +[metadata] +description_file=README.md + +[flake8] +application-import-names = artifactory_catalog_connector +application-package-names = artifactory_catalog_connector +enable-extensions = G +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # File contains nothing but comments + H104, + # Include name with TODOs as in # TODO(yourname) + H101, + # Enable mocking + H216, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405, + # Allow breaks after binary operators + W504 +import-order-style = google +max-line-length = 120 diff --git a/component-catalog-connectors/artifactory-connector/setup.py b/component-catalog-connectors/artifactory-connector/setup.py new file mode 100644 index 0000000..e747acd --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/setup.py @@ -0,0 +1,69 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from setuptools import find_packages, setup + +long_desc = """ + Elyra component catalog connector for Artifactory + """ + +here = os.path.abspath(os.path.dirname(__file__)) + +version_ns = {} +with open(os.path.join(here, "artifactory_catalog_connector", "_version.py")) as f: + exec(f.read(), {}, version_ns) + +setup_args = dict( + name="elyra-artifactory-catalog-connector", + version=version_ns["__version__"], + url="https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/artifactory-connector", + description="Elyra component catalog connector for Artifactory", + long_description=long_desc, + author="Elyra Maintainers", + license="Apache License Version 2.0", + packages=find_packages(), + install_requires=[], + setup_requires=["flake8"], + include_package_data=True, + entry_points={ + "metadata.schemas_providers": [ + "artifactory-catalog-schema = " + "artifactory_catalog_connector.artifactory_schema_provider:ArtifactorySchemasProvider" + ], + "elyra.component.catalog_types": [ + "artifactory-catalog = " + "artifactory_catalog_connector.artifactory_catalog_connector:ArtifactoryComponentCatalogConnector" + ], + }, + classifiers=[ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], +) + +if __name__ == "__main__": + setup(**setup_args) diff --git a/component-catalog-connectors/artifactory-connector/test_requirements.txt b/component-catalog-connectors/artifactory-connector/test_requirements.txt new file mode 100644 index 0000000..28f28b5 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/test_requirements.txt @@ -0,0 +1 @@ +flake8>=3.5.0,<3.9.0 diff --git a/component-catalog-connectors/connector-directory.md b/component-catalog-connectors/connector-directory.md index a2f32f8..dc64e7d 100644 --- a/component-catalog-connectors/connector-directory.md +++ b/component-catalog-connectors/connector-directory.md @@ -5,9 +5,10 @@ The following catalog connectors should work with Elyra version 3.3 and above. C - To add your connector to the list [create a pull request](https://github.com/elyra-ai/examples/pulls). - Learn [how to build your own catalog connector](build-a-custom-connector.md). -| Connector | Description | -| ----------- | ----------- | +| Connector | Description | +| --- | --- | | [Apache Airflow example catalog](airflow-example-components-connector) | Provides access to a small set of curated Apache Airflow operators that you can use to get started with the Visual Pipeline Editor. | | [Kubeflow Pipelines example catalog](kfp-example-components-connector) | Provides access to a small set of curated Kubeflow Pipelines components that you can use to get started with the Visual Pipeline Editor. | +| [Artifactory](artifactory-connector) | Enables Elyra to load pipelines components from a generic-type Artifactory repo. | | [Machine Learning Exchange](mlx-connector/) | This LFAI project provides an open source Data and AI assets catalog and execution engine for Kubeflow Pipelines. | From e0b074c625e7931fa1fa89cba534859717a9e157 Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Wed, 1 Jun 2022 15:13:48 +1000 Subject: [PATCH 02/10] Support Elyra 3.7 --- .../artifactory_catalog_connector.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py index 0180fed..a25715f 100644 --- a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py @@ -23,7 +23,7 @@ import requests from artifactory_catalog_connector import packaging_ports -from elyra.pipeline.catalog_connector import ComponentCatalogConnector +from elyra.pipeline.catalog_connector import ComponentCatalogConnector, EntryData, KfpEntryData from requests.auth import HTTPBasicAuth, AuthBase @@ -235,20 +235,19 @@ def get_catalog_entries( return component_list - def read_catalog_entry( + def get_entry_data( self, catalog_entry_data: Dict[str, Any], catalog_metadata: Dict[str, Any] - ) -> Optional[str]: + ) -> Optional[EntryData]: """ - Fetch the component that is identified by catalog_entry_data from - the Artifactory catalog. + Fetch the component that is identified by catalog_entry_data from the Artifactory catalog. :param catalog_entry_data: a dictionary that contains the information needed to read the content of the component definition :param catalog_metadata: the metadata associated with the catalog in which this catalog entry is stored; in addition to catalog_entry_data, catalog_metadata may also be needed to read the component definition for certain types of catalogs - - :returns: the content of the given catalog entry's definition in string form + :returns: an EntryData object representing the definition (and other identifying info) for a single + catalog entry; if None is returned, this catalog entry is skipped and a warning message logged """ artifactory_username = catalog_metadata.get("artifactory_username") artifactory_password = catalog_metadata.get("artifactory_password") @@ -280,11 +279,12 @@ def read_catalog_entry( ) return None - return resp.text + return KfpEntryData(definition=resp.text) - def get_hash_keys(self) -> List[Any]: + @classmethod + def get_hash_keys(cls) -> List[Any]: """ - Identifies the unique Artifactory catalog key that method read_catalog_entry + Identifies the unique Artifactory catalog key that get_entry_data can use to fetch a component from the catalog. Method get_catalog_entries retrieves the list of available key values from the catalog. From 1c5ff9d80c5a9fa2d5a82619189984176f38a277 Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Thu, 2 Jun 2022 12:43:04 +1000 Subject: [PATCH 03/10] Add tests and fix small issues --- .../artifactory-connector/Makefile | 5 +- .../artifactory_catalog_connector.py | 172 +++++++-- .../test_requirements.txt | 2 + .../tests/resources/component-template.yaml | 27 ++ .../tests/test_connector.py | 325 ++++++++++++++++++ 5 files changed, 499 insertions(+), 32 deletions(-) create mode 100644 component-catalog-connectors/artifactory-connector/tests/resources/component-template.yaml create mode 100644 component-catalog-connectors/artifactory-connector/tests/test_connector.py diff --git a/component-catalog-connectors/artifactory-connector/Makefile b/component-catalog-connectors/artifactory-connector/Makefile index d98abb4..4f25c07 100644 --- a/component-catalog-connectors/artifactory-connector/Makefile +++ b/component-catalog-connectors/artifactory-connector/Makefile @@ -15,7 +15,7 @@ # .PHONY: help clean lint test-dependencies source-install install dist -.PHONY: publish test-publish +.PHONY: publish test-publish test SHELL:=/bin/bash @@ -47,6 +47,9 @@ source-install: dist ## Install component connector package from source install: ## Install component connector package from PyPI pip install $(PACKAGE_NAME) +test: source-install ## Run automated tests + pytest tests/ + test-publish: dist ## Upload package to test PyPI twine upload --repository testpypi dist/* diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py index a25715f..5cebd32 100644 --- a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory_catalog_connector.py @@ -14,6 +14,7 @@ # limitations under the License. # import fnmatch +import re from http import HTTPStatus from typing import Any from typing import Dict @@ -23,7 +24,11 @@ import requests from artifactory_catalog_connector import packaging_ports -from elyra.pipeline.catalog_connector import ComponentCatalogConnector, EntryData, KfpEntryData +from elyra.pipeline.catalog_connector import ( + ComponentCatalogConnector, + EntryData, + KfpEntryData, +) from requests.auth import HTTPBasicAuth, AuthBase @@ -62,19 +67,23 @@ def get_folder_info( req_url = f"{api_base_url}/api/storage/{repository_name}/{folder_path.strip('/')}" resp = requests.get(req_url, auth=api_auth) + # verify HTTP status code if resp.status_code != HTTPStatus.OK: raise RuntimeError( f"Failed to get FolderInfo from '{req_url}'... " f"Got unhandled HTTP Status {resp.status_code}, expected 200... " f"{resp.text}" ) + + # verify Content-Type + resp_content_type = resp.headers.get("Content-Type", "") expected_content_type = ( "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json" ) - if resp.headers["Content-Type"] != expected_content_type: + if resp_content_type != expected_content_type: raise RuntimeError( f"Failed to get FolderInfo from '{req_url}'... " - f"Got unexpected Content-Type '{resp.headers['Content-Type']}', " + f"Got unexpected Content-Type '{resp_content_type}', " f"expected '{expected_content_type}'" ) @@ -192,19 +201,94 @@ def get_catalog_entries( """ component_list = [] - artifactory_url = catalog_metadata["artifactory_url"] + ######################## + # required metadata + ######################## + artifactory_url = catalog_metadata.get("artifactory_url") + if artifactory_url is None: + self.log.error("Artifactory catalogs must specify `artifactory_url`") + return component_list + + repository_name = catalog_metadata.get("repository_name") + if artifactory_url is None: + self.log.error("Artifactory catalogs must specify `repository_name`") + return component_list + + repository_path = catalog_metadata.get("repository_path") + if repository_path is None: + self.log.error("Artifactory catalogs must specify `repository_path`") + return component_list + + max_recursion_depth = catalog_metadata.get("max_recursion_depth") + if max_recursion_depth is None: + self.log.error("Artifactory catalogs must specify `max_recursion_depth`") + return component_list + + max_files_per_folder = catalog_metadata.get("max_files_per_folder") + if max_files_per_folder is None: + self.log.error("Artifactory catalogs must specify `max_files_per_folder`") + return component_list + + file_filter = catalog_metadata.get("file_filter") + if file_filter is None: + self.log.error("Artifactory catalogs must specify `file_filter`") + return component_list + + file_ordering = catalog_metadata.get("file_ordering") + if file_ordering is None: + self.log.error("Artifactory catalogs must specify `file_ordering`") + return component_list + + # parse `artifactory_url` + url_obj = urlparse(artifactory_url) + url_path = url_obj.path.strip("/") + if url_path: + api_base_url = f"{url_obj.scheme}://{url_obj.netloc}/{url_path}" + else: + api_base_url = f"{url_obj.scheme}://{url_obj.netloc}" + + # parse `max_recursion_depth` + _max_recursion_depth_regex = re.compile(r"^[0-9]+$") + if _max_recursion_depth_regex.match(max_recursion_depth): + max_recursion_depth = int(max_recursion_depth) + else: + self.log.error( + f"`max_recursion_depth` in Artifactory catalogs must match regex: " + f"{_max_recursion_depth_regex.pattern}" + ) + return component_list + + # parse `max_files_per_folder` + _max_files_per_folder_regex = re.compile(r"^-1|[0-9]+$") + if _max_files_per_folder_regex.match(max_files_per_folder): + max_files_per_folder = int(max_files_per_folder) + else: + self.log.error( + f"`max_files_per_folder` in Artifactory catalogs must match regex: " + f"{_max_files_per_folder_regex.pattern}" + ) + return component_list + + # parse `file_ordering` + _file_ordering_options = [ + "NAME_ASCENDING", + "NAME_DESCENDING", + "VERSION_ASCENDING", + "VERSION_DESCENDING", + ] + if file_ordering not in _file_ordering_options: + self.log.error( + f"`file_ordering` in Artifactory catalogs must be one of: {_file_ordering_options}" + ) + return component_list + + ######################## + # optional metadata + ######################## artifactory_username = catalog_metadata.get("artifactory_username") artifactory_password = catalog_metadata.get("artifactory_password") - repository_name = catalog_metadata["repository_name"] - repository_path = catalog_metadata["repository_path"] - max_recursion_depth = int(catalog_metadata["max_recursion_depth"]) - max_files_per_folder = int(catalog_metadata["max_files_per_folder"]) - file_filter = catalog_metadata["file_filter"] - file_ordering = catalog_metadata["file_ordering"] - - url_obj = urlparse(artifactory_url) - api_base_url = f"{url_obj.scheme}://{url_obj.netloc}/{url_obj.path.strip('/')}" + # parse `artifactory_username` and `artifactory_password` api_auth = None if artifactory_username and artifactory_password: api_auth = HTTPBasicAuth( @@ -213,9 +297,13 @@ def get_catalog_entries( elif artifactory_password: api_auth = ArtifactoryApiKeyAuth(api_key=artifactory_password) + ######################## + # get component list + ######################## try: - self.log.debug("Retrieving component list from Artifactory catalog.") - + self.log.debug( + f"Retrieving component list from Artifactory catalog '{artifactory_url}'" + ) component_list += recursively_get_components( api_base_url=api_base_url, api_auth=api_auth, @@ -227,10 +315,9 @@ def get_catalog_entries( current_folder_path=repository_path, current_recursion_depth=0, ) - except Exception as ex: self.log.error( - f"Error retrieving component list from Artifactory catalog: {ex}" + f"Error retrieving component list from Artifactory catalog '{artifactory_url}': {ex}" ) return component_list @@ -249,14 +336,21 @@ def get_entry_data( :returns: an EntryData object representing the definition (and other identifying info) for a single catalog entry; if None is returned, this catalog entry is skipped and a warning message logged """ + ######################## + # required metadata + ######################## + entry_url = catalog_entry_data.get("url") + if entry_url is None: + self.log.error("Artifactory component entries must specify `url`") + return None + + ######################## + # optional metadata + ######################## artifactory_username = catalog_metadata.get("artifactory_username") artifactory_password = catalog_metadata.get("artifactory_password") - url = catalog_entry_data.get("url") - if url is None: - self.log.error("Artifactory component source must contain `url`.") - return None - + # parse `artifactory_username` and `artifactory_password` api_auth = None if artifactory_username and artifactory_password: api_auth = HTTPBasicAuth( @@ -265,17 +359,33 @@ def get_entry_data( elif artifactory_password: api_auth = ArtifactoryApiKeyAuth(api_key=artifactory_password) - resp = requests.get(url, auth=api_auth) - if resp.status_code != HTTPStatus.OK: - self.log.error( - f"Failed to read component from '{url}'... " - f"Got unhandled HTTP Status {resp.status_code}, expected 200" + ######################## + # get component + ######################## + try: + self.log.debug( + f"Retrieving component from Artifactory catalog '{entry_url}'" ) - return None - if resp.headers["Content-Type"] != "text/plain": + resp = requests.get(entry_url, auth=api_auth) + + # verify HTTP status code + if resp.status_code != HTTPStatus.OK: + raise RuntimeError( + f"Got unhandled HTTP Status {resp.status_code}, expected 200... " + f"{resp.text}" + ) + + # verify Content-Type + resp_content_type = resp.headers.get("Content-Type", "") + expected_content_type = "text/plain" + if resp_content_type != expected_content_type: + raise RuntimeError( + f"Got unexpected Content-Type '{resp_content_type}', " + f"expected '{expected_content_type}'" + ) + except Exception as ex: self.log.error( - f"Failed to read component from '{url}'... " - f"Got unexpected Content-Type '{resp.headers['Content-Type']}', expected 'text/plain'" + f"Error retrieving component specification from '{entry_url}': {ex}" ) return None diff --git a/component-catalog-connectors/artifactory-connector/test_requirements.txt b/component-catalog-connectors/artifactory-connector/test_requirements.txt index 28f28b5..6708441 100644 --- a/component-catalog-connectors/artifactory-connector/test_requirements.txt +++ b/component-catalog-connectors/artifactory-connector/test_requirements.txt @@ -1 +1,3 @@ flake8>=3.5.0,<3.9.0 +pytest +requests-mock \ No newline at end of file diff --git a/component-catalog-connectors/artifactory-connector/tests/resources/component-template.yaml b/component-catalog-connectors/artifactory-connector/tests/resources/component-template.yaml new file mode 100644 index 0000000..30aac77 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/tests/resources/component-template.yaml @@ -0,0 +1,27 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: {{COMPONENT_NAME}} - {{COMPONENT_VERSION}} +description: {{COMPONENT_NAME}} +inputs: + - name: print_text + type: String + default: "Hello World!" +implementation: + container: + image: alpine:latest + args: + - echo + - { inputValue: print_text } \ No newline at end of file diff --git a/component-catalog-connectors/artifactory-connector/tests/test_connector.py b/component-catalog-connectors/artifactory-connector/tests/test_connector.py new file mode 100644 index 0000000..0a77f18 --- /dev/null +++ b/component-catalog-connectors/artifactory-connector/tests/test_connector.py @@ -0,0 +1,325 @@ +# +# Copyright 2018-2022 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +from typing import Dict, Union + +import pytest +from artifactory_catalog_connector.artifactory_catalog_connector import ( + ArtifactoryComponentCatalogConnector, +) +from elyra.pipeline.catalog_connector import KfpEntryData +from requests.exceptions import ConnectTimeout +from requests_mock import Mocker + + +################################ +# TEST HELPERS +################################ +def _component_yaml(component_name: str, component_version: str) -> str: + with open("tests/resources/component-template.yaml", "r") as f: + return ( + f.read() + .replace("{{COMPONENT_NAME}}", component_name) + .replace("{{COMPONENT_VERSION}}", component_version) + ) + + +def _register_artifactory_mocks( + requests_mock: Mocker, + api_base_url: str, + repository_name: str, + folder_path: str, + folder_content: Dict[str, Union[Dict, str]], +): + for name, content in folder_content.items(): + # CASE: content represents a file + if isinstance(content, str): + file_path = f"{folder_path}/{name}" + + # register a mock that returns the file content + url = f"{api_base_url.rstrip('/')}/{repository_name}/{file_path.strip('/')}" + requests_mock.get( + url=url, + headers={"Content-Type": "text/plain"}, + text=content, + ) + logging.getLogger().warning(f"Registered Mock URL - {url}") + + # CASE: content represents a folder + elif isinstance(content, dict): + subfolder_path = f"{folder_path}/{name}" + + # register a mock that returns the FolderInfo + # https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-FolderInfo + url = f"{api_base_url.rstrip('/')}/api/storage/{repository_name}/{subfolder_path.strip('/')}" + requests_mock.get( + url=url, + headers={ + "Content-Type": "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json" + }, + json={ + "uri": url, + "repo": repository_name, + "path": f"/{subfolder_path.strip('/')}", + "created": "2000-01-01T00:00:00.000Z", + "createdBy": "userY", + "lastModified": "2000-01-01T00:00:00.000Z", + "modifiedBy": "userY", + "lastUpdated": "2000-01-01T00:00:00.000Z", + "children": [ + { + "uri": f"/{child_name}", + "folder": isinstance(child_content, dict), + } + for child_name, child_content in content.items() + ], + }, + ) + logging.getLogger().warning(f"Registered Mock URL - {url}") + + # recursively call this function to populate subdirectories + _register_artifactory_mocks( + requests_mock=requests_mock, + api_base_url=api_base_url, + repository_name=repository_name, + folder_path=f"{folder_path}/{name}", + folder_content=content, + ) + + +################################ +# TESTS +################################ +class TestArtifactoryComponentCatalogConnector: + + log = logging.getLogger(__name__) + + @pytest.fixture(scope="function") + def connector(self): + _kfp_supported_file_types = [".yaml"] + yield ArtifactoryComponentCatalogConnector(_kfp_supported_file_types) + + @pytest.fixture(scope="function") + def mock_artifactory(self, requests_mock): + _artifactory_url = "https://artifactory.example.com/" + _repository_name = "my-repository" + _repo_content = { + "components": { + "component_1": { + "__COMPONENT__": "", + "component-1.0.9.yaml": _component_yaml("component_1", "1.0.9"), + "component-1.0.10.yaml": _component_yaml("component_1", "1.0.10"), + }, + "component_2": { + "__COMPONENT__": "", + "component-1.0.0.yaml": _component_yaml("component_2", "1.0.0"), + "component-1.1.0.yaml": _component_yaml("component_2", "1.1.0"), + }, + "component_3": { + "__COMPONENT__": "", + "component-1.0.0.yaml": _component_yaml("component_3", "1.0.0"), + "component-1.1.0.yaml": _component_yaml("component_3", "1.1.0"), + "hidden_component": { + "__COMPONENT__": "", + "component-1.0.0.yaml": _component_yaml( + "hidden_component", "1.0.0" + ), + "component-1.1.0.yaml": _component_yaml( + "hidden_component", "1.1.0" + ), + }, + }, + } + } + _register_artifactory_mocks( + requests_mock=requests_mock, + api_base_url=_artifactory_url, + repository_name=_repository_name, + folder_path="", + folder_content=_repo_content, + ) + yield _artifactory_url, _repository_name + + def test__get_hash_keys(self, connector): + """ + Verify that `get_hash_keys()` returns the expected hash keys + """ + hash_keys = connector.get_hash_keys() + assert len(hash_keys) == 1 + assert hash_keys[0] == "url" + + def test__get_catalog_entries__invalid(self, connector, requests_mock): + """ + Test various invalid `get_catalog_entries()` scenarios + """ + _artifactory_url = "https://invalid.example.com/" + _repository_name = "my-repository" + _repository_path = "/components" + _catalog_metadata = { + "artifactory_url": _artifactory_url, + "repository_name": _repository_name, + "repository_path": _repository_path, + "max_recursion_depth": "3", + "max_files_per_folder": "-1", + "file_filter": "*.yaml", + "file_ordering": "VERSION_ASCENDING", + } + + # the URL of the expected first API call given the above parameters + _storage_api_url = ( + f"{_artifactory_url.rstrip('/')}" + f"/api/storage" + f"/{_repository_name}" + f"/{_repository_path.strip('/')}" + ) + + # TEST - the specified URL times out + mock = requests_mock.get(url=_storage_api_url, exc=ConnectTimeout) + catalog_entries = connector.get_catalog_entries(_catalog_metadata) + assert mock.called + assert len(catalog_entries) == 0 + + # TEST - the specified URL returns non-200 status code + mock = requests_mock.get(url=_storage_api_url, status_code=400) + catalog_entries = connector.get_catalog_entries(_catalog_metadata) + assert mock.called + assert len(catalog_entries) == 0 + + # TEST - the specified URL returns wrong content type 'plain/text' + mock = requests_mock.get( + url=_storage_api_url, + text="some random text", + headers={"Content-Type": "plain/text"}, + ) + catalog_entries = connector.get_catalog_entries(_catalog_metadata) + assert mock.called + assert len(catalog_entries) == 0 + + # TEST - the specified URL returns correct content type, but malformed + mock = requests_mock.get( + url=_storage_api_url, + json={"children": [{"invalid_key": "some value"}]}, + headers={ + "Content-Type": "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json" + }, + ) + catalog_entries = connector.get_catalog_entries(_catalog_metadata) + assert mock.called + assert len(catalog_entries) == 0 + + def test__get_catalog_entries__valid(self, connector, mock_artifactory): + """ + Test various valid `get_catalog_entries()` scenarios + """ + _artifactory_url, _repository_name = mock_artifactory + + # TEST - single component folder + catalog_entries = connector.get_catalog_entries( + { + "artifactory_url": _artifactory_url, + "repository_name": _repository_name, + "repository_path": "/components/component_1", + "max_recursion_depth": "3", + "max_files_per_folder": "-1", + "file_filter": "*.yaml", + "file_ordering": "VERSION_ASCENDING", + } + ) + expected_catalog_entries = [ + {"url": f"{_artifactory_url.rstrip('/')}/{_repository_name}/{path}"} + for path in [ + "components/component_1/component-1.0.9.yaml", + "components/component_1/component-1.0.10.yaml", + ] + ] + assert catalog_entries == expected_catalog_entries + + # TEST - multiple component folders + catalog_entries = connector.get_catalog_entries( + { + "artifactory_url": _artifactory_url, + "repository_name": _repository_name, + "repository_path": "/components", + "max_recursion_depth": "3", + "max_files_per_folder": "-1", + "file_filter": "*.yaml", + "file_ordering": "VERSION_ASCENDING", + } + ) + expected_catalog_entries = [ + {"url": f"{_artifactory_url.rstrip('/')}/{_repository_name}/{path}"} + for path in [ + "components/component_1/component-1.0.9.yaml", + "components/component_1/component-1.0.10.yaml", + "components/component_2/component-1.0.0.yaml", + "components/component_2/component-1.1.0.yaml", + "components/component_3/component-1.0.0.yaml", + "components/component_3/component-1.1.0.yaml", + ] + ] + assert catalog_entries == expected_catalog_entries + + def test__get_entry_data__invalid(self, connector, requests_mock): + """ + Test various invalid `get_entry_data()` scenarios + """ + _entry_url = f"https://invalid.example.com/my-repository/component.yaml" + _catalog_entry_data = {"url": _entry_url} + _catalog_metadata = {} + + # TEST - the specified URL times out + mock = requests_mock.get(url=_entry_url, exc=ConnectTimeout) + entry_data = connector.get_entry_data(_catalog_entry_data, _catalog_metadata) + assert mock.called + assert entry_data is None + + # TEST - the specified URL returns non-200 status code + mock = requests_mock.get(url=_entry_url, status_code=400) + entry_data = connector.get_entry_data(_catalog_entry_data, _catalog_metadata) + assert mock.called + assert entry_data is None + + # TEST - the specified URL returns wrong content type (not 'plain/text') + mock = requests_mock.get( + url=_entry_url, + json={"some key": "some value"}, + headers={"Content-Type": "application/json"}, + ) + entry_data = connector.get_entry_data(_catalog_entry_data, _catalog_metadata) + assert mock.called + assert entry_data is None + + def test__get_entry_data__valid(self, connector, mock_artifactory): + """ + Test various valid `get_entry_data()` scenarios + """ + _artifactory_url, _repository_name = mock_artifactory + + # TEST - single component definition + entry_data = connector.get_entry_data( + catalog_entry_data={ + "url": ( + f"{_artifactory_url.rstrip('/')}" + f"/{_repository_name}" + f"/components/component_1/component-1.0.10.yaml" + ) + }, + # we don't use catalog_metadata when retrieving entries, so can be blank + catalog_metadata={}, + ) + assert isinstance(entry_data, KfpEntryData) + assert entry_data.definition == _component_yaml("component_1", "1.0.10") From 15bbdfd29d2e4a91656cc67dcf5bb089af8dcce2 Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Thu, 2 Jun 2022 12:46:27 +1000 Subject: [PATCH 04/10] Fix flake8 compliance --- .../artifactory-connector/tests/test_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component-catalog-connectors/artifactory-connector/tests/test_connector.py b/component-catalog-connectors/artifactory-connector/tests/test_connector.py index 0a77f18..726510f 100644 --- a/component-catalog-connectors/artifactory-connector/tests/test_connector.py +++ b/component-catalog-connectors/artifactory-connector/tests/test_connector.py @@ -315,7 +315,7 @@ def test__get_entry_data__valid(self, connector, mock_artifactory): "url": ( f"{_artifactory_url.rstrip('/')}" f"/{_repository_name}" - f"/components/component_1/component-1.0.10.yaml" + "/components/component_1/component-1.0.10.yaml" ) }, # we don't use catalog_metadata when retrieving entries, so can be blank From 2f10d82c9175ad3721daae74ae562c1b66daf7cd Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Thu, 2 Jun 2022 12:53:27 +1000 Subject: [PATCH 05/10] Fix flake8 compliance --- .../artifactory-connector/tests/test_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/tests/test_connector.py b/component-catalog-connectors/artifactory-connector/tests/test_connector.py index 726510f..056acd5 100644 --- a/component-catalog-connectors/artifactory-connector/tests/test_connector.py +++ b/component-catalog-connectors/artifactory-connector/tests/test_connector.py @@ -182,7 +182,7 @@ def test__get_catalog_entries__invalid(self, connector, requests_mock): # the URL of the expected first API call given the above parameters _storage_api_url = ( f"{_artifactory_url.rstrip('/')}" - f"/api/storage" + "/api/storage" f"/{_repository_name}" f"/{_repository_path.strip('/')}" ) @@ -277,7 +277,7 @@ def test__get_entry_data__invalid(self, connector, requests_mock): """ Test various invalid `get_entry_data()` scenarios """ - _entry_url = f"https://invalid.example.com/my-repository/component.yaml" + _entry_url = "https://invalid.example.com/my-repository/component.yaml" _catalog_entry_data = {"url": _entry_url} _catalog_metadata = {} From d3327c3df68c8539763cdf720ed9b2ae8aa44127 Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Fri, 3 Jun 2022 15:29:36 +1000 Subject: [PATCH 06/10] update readme --- .../artifactory-connector/README.md | 231 ++++++++++++------ 1 file changed, 154 insertions(+), 77 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/README.md b/component-catalog-connectors/artifactory-connector/README.md index 537d0e1..6c54b6d 100644 --- a/component-catalog-connectors/artifactory-connector/README.md +++ b/component-catalog-connectors/artifactory-connector/README.md @@ -4,30 +4,30 @@ This catalog connector enables Elyra to load pipelines components from a generic ## Install the connector -You can install the Artifactory catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. +You can install the Artifactory catalog connector from PyPI or source code. ### Prerequisites -- [Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above) -- A [generic-type Artifactory repo](https://www.jfrog.com/confluence/display/JFROG/Repository+Management#RepositoryManagement-GenericRepositories) +- [Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.7 and above) +- A [generic-type](https://www.jfrog.com/confluence/display/JFROG/Repository+Management#RepositoryManagement-GenericRepositories) Artifactory repo ### Install from PyPI - ``` - $ pip install elyra-artifactory-catalog-connector - ``` +```bash +$ pip install elyra-artifactory-catalog-connector +``` ### Install from source code - ``` - $ git clone https://github.com/elyra-ai/examples.git - $ cd examples/component-catalog-connectors/artifactory-connector/ - $ make source-install - ``` +```bash +$ git clone https://github.com/elyra-ai/examples.git +$ cd examples/component-catalog-connectors/artifactory-connector/ +$ make source-install +``` -## Use the connector +## Create a Catalog -### Add a new Artifactory catalog +### From the UI 1. Launch JupyterLab 2. Open the [`Manage Components` panel](https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui) @@ -48,16 +48,92 @@ You can install the Artifactory catalog connector from PyPI or source code. Note 8. Save the catalog entry 9. Open the Visual Pipeline Editor and expand the palette. The catalog components are displayed. -### Recommended Usage +### From the CLI + +```bash +$ elyra-metadata install component-catalogs \ + --schema_name="artifactory-catalog" \ + --name="my_artifactory" \ + --display_name="My Artifactory" \ + --description="components from my artifactory" \ + --runtime_type="KUBEFLOW_PIPELINES" \ + --categories='["My Artifactory"]' \ + --artifactory_url="https://example.com/artifactory/" \ + --repository_name="my-kubeflow-components" \ + --repository_path="/" \ + --max_recursion_depth="3" \ + --max_files_per_folder="-1" \ + --file_filter="*.yaml" \ + --file_ordering="VERSION_DESCENDING" +``` + +## Documentation -We recommend that you version your component YAML rather than having users always point to the latest version of each component. -Otherwise, when changes are made to `inputs`/`outputs`/`implementation`, existing pipelines may break or do unexpected things. +### Introduction + +The basic idea, is to create a [generic-type](https://www.jfrog.com/confluence/display/JFROG/Repository+Management#RepositoryManagement-GenericRepositories) +Artifactory repo with a folder structure similar to the following: + +``` +my_component_1/ + __COMPONENT__ + 1.0.0.yaml + 1.1.0.yaml + 2.0.0.yaml +my_component_2/ + __COMPONENT__ + 1.0.0.yaml + 1.1.0.yaml + 2.0.0.yaml +special_components/ + special_component_1/ + __COMPONENT__ + 1.0.0.yaml + 1.1.0.yaml + 2.0.0.yaml + special_component_2/ + __COMPONENT__ + 1.0.0.yaml + 1.1.0.yaml + 2.0.0.yaml +``` + +The most important thing to understand are the `__COMPONENT__` marker files, which tell the connector that a folder contains a component. + +1. The content of the `__COMPONENT__` files is not considered, only the fact that they exist. +2. The presence of a `__COMPONENT__` file in a folder prevents further recursion into sub-folders of that folder. + +### Catalog Configs + +| Name | CLI Parameter | Description | Example | Required | +|------------------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------|----------| +| Artifactory URL | `artifactory_url` | URL of the Artifactory server | `https://example.com/artifactory/` | YES | +| Artifactory Username | `artifactory_username` | Username for the Artifactory server | `service_user` | NO | +| Artifactory Password/API-Key | `artifactory_password` | Password or API Key for the Artifactory server | `password123` | NO | +| Repository Name | `repository_name` | Name of the Artifactory repository | `my-kubeflow-components` | YES | +| Repository Path | `repository_path` | Path of folder in repository to search under | `/path/to/components/` | YES | +| Maximum Recursion Depth | `max_recursion_depth` | Maximum folder depth to recurse looking for `__COMPONENT__` marker files | `3` | YES | +| Maximum Files Per Folder | `max_files_per_folder` | Maximum number of files returned from each folder. ('-1' is unlimited) | `-1` | YES | +| File Filter | `file_filter` | Fnmatch file name filter.
`*` match everything;
`?` any single character;
`[seq]` character in seq;
`[!seq]` character not in seq;
`[0-9]` any single number; | `*.yaml` | YES | +| File Ordering | `file_ordering` | Order in which files are processed per folder.
`NAME_ASCENDING` alphanumeric ascending;
`NAME_DESCENDING` alphanumeric descending;
`VERSION_ASCENDING` packaging.version.LegacyVersion() ascending;
`VERSION_DESCENDING` packaging.version.LegacyVersion() descending; | `VERSION_DESCENDING` | YES | + +### Component Versioning + +Kubeflow has no native concept of "versioning" the same Component YAML over time ([proposal to add one](https://github.com/kubeflow/pipelines/issues/7832)). + +If changes are made to a Component's YAML, existing pipelines may break or do unexpected things. +Given this, we recommend __treating each Component YAML as atomic__, and creating a new YAML file for every change (never updating the old ones). + +When doing this, using [Semantic Versions](https://semver.org/) for file names is sensible. +For example, you may initially create a file called `0.1.0.yaml`, then make a change and create a new file called `X.X.X.yaml` +(the appropriate version number depends on if the change was backwards-compatible or not). > 🟦 __Tip__ 🟦 > -> Include the version in each component's `name` so they can be distinguished in the Elyra UI. -> -> For example: +> Make sure to include the version in each component's `name` so they can be distinguished in the Elyra UI. +> +> For example, this component has `1.0.0` in its `name`: +> > ```yaml > name: My Component - 1.0.0 > description: "" @@ -66,51 +142,51 @@ Otherwise, when changes are made to `inputs`/`outputs`/`implementation`, existin > implementation: {} > ``` -Elyra only stores pointers to the source-catalog of each component in its `.pipeline` files. -This means your catalogs must continue to include ALL old component versions, otherwise old pipelines will be unable to run. - -You can solve this problem with the `Artifactory Catalog Connector` by adding TWO catalog instances for your single Artifactory repo: - -1. that has only the latest version of each component: - - `max_files_per_folder = 1` - - `file_ordering = VERSION_DESCENDING` -2. that has all versions of each component: - - `max_files_per_folder = -1` - -> 🟦 __Tip__ 🟦 +> 🟥 __Warning__ 🟥 > -> Name your component YAML files with consistent prefixes before the version. -> -> This is because `VERSION_DESCENDING` treats the whole file-name as a version. -> For example, `aaaa-1.0.0.yaml` would be sorted before `bbbb-9.0.0.yaml`. +> For Elyra to run a pipeline with components from a catalog, +> those components must be present in one of your catalogs. > -> Alternatively, don't include any prefix on file names, for example `1.0.0.yaml`, `1.1.0.yaml`. - +> This means your catalogs must continue to include ALL old component versions, +> otherwise pipelines using old components will be unable to run. +> +> To keep the UI clean, you may want to create TWO catalog instances: +> +> 1. that ONLY has the LATEST version of each component: +> - set `max_files_per_folder` to `1` +> - set `file_ordering` to `VERSION_DESCENDING` +> 2. that has ALL versions of each component: +> - set `max_files_per_folder` to `-1` -### Example Configs +## Examples Assume an artifactory server `https://example.com/artifactory/` has a repository called `elyra-components` with the following folder structure: ``` -component_1/ +component_one/ __COMPONENT__ - component-1.0.9.yaml - component-1.0.10.yaml -component_2/ - hidden_component/ + 1.0.0.yaml + 2.0.0.yaml +missing_marker/ + 1.0.0.yaml + 2.0.0.yaml +component_group/ + component_two/ __COMPONENT__ - component-1.0.0.yaml - component-1.1.0.yaml - __COMPONENT__ - component-1.0.0.yaml - component-1.1.0.yaml -component_3/ - component-1.0.0.yaml - component-1.1.0.yaml + 1.0.0.yaml + 2.0.0.yaml + nested_component/ + __COMPONENT__ + 1.0.0.yaml + 2.0.0.yaml + component_three/ + __COMPONENT__ + 1.0.0.yaml + 2.0.0.yaml ``` -__Example 1:__ +### Example 1 ``` Configs: @@ -125,18 +201,20 @@ file_ordering = VERSION_DESCENDING Matched: ========= -https://example.com/artifactory/elyra-components/component_1/component-1.0.9.yaml -https://example.com/artifactory/elyra-components/component_1/component-1.0.10.yaml -https://example.com/artifactory/elyra-components/component_2/component-1.0.0.yaml -https://example.com/artifactory/elyra-components/component_2/component-1.1.0.yaml +/component_one/1.0.0.yaml +/component_one/2.0.0.yaml +/component_group/component_two/1.0.0.yaml +/component_group/component_two/2.0.0.yaml +/component_group/component_three/1.0.0.yaml +/component_group/component_three/2.0.0.yaml Notes: ========= -- the `component_3/` files are not matched as this folder does not contain a `__COMPONENT__` marker -- the `component_2/hidden_component/` files are not matched as recursion stops at the first `__COMPONENT__` marker +- the `/missing_marker` files are not matched as this folder does not contain a `__COMPONENT__` marker +- the `/component_group/component_two/nested_component` files are not matched as recursion stops at the first `__COMPONENT__` marker ``` -__Example 2:__ +### Example 2 ``` Configs: @@ -144,21 +222,28 @@ Configs: artifactory_url = https://example.com/artifactory/ repository_name = elyra-components repository_path = / -max_recursion_depth = 0 -max_files_per_folder = -1 +max_recursion_depth = 3 +max_files_per_folder = 1 file_filter = *.yaml file_ordering = VERSION_DESCENDING Matched: ========= -N/A +/component_one/2.0.0.yaml +/component_group/component_two/2.0.0.yaml +/component_group/component_three/2.0.0.yaml Notes: ========= -- no files are matched, as `max_recursion_depth` is `0` +- as `max_files_per_folder` is `1`, only ONE file from each folder is matched +- as `file_ordering` is `VERSION_DESCENDING`, the HIGHEST version file is returned + - we use `packaging.version.LegacyVersion()` to preform the sort + - the `file_ordering` is applied separately within each folder + - the WHOLE file-name is treated as a version, so "aaaa-1.0.0.yaml" would be sorted before "bbbb-9.0.0.yaml" + (to avoid this problem, don't include any prefix on file names, for example `1.0.0.yaml`, `1.1.0.yaml`) ``` -__Example 3:__ +### Example 3 ``` Configs: @@ -166,24 +251,18 @@ Configs: artifactory_url = https://example.com/artifactory/ repository_name = elyra-components repository_path = / -max_recursion_depth = 3 -max_files_per_folder = 1 +max_recursion_depth = 0 +max_files_per_folder = -1 file_filter = *.yaml file_ordering = VERSION_DESCENDING Matched: ========= -https://example.com/artifactory/elyra-components/component_1/component-1.0.10.yaml -https://example.com/artifactory/elyra-components/component_2/component-1.1.0.yaml +NONE Notes: ========= -- the `file_ordering` is applied separately within each folder -- as `max_files_per_folder` is `1`, only ONE file from each folder is matched -- as `file_ordering` is `VERSION_DESCENDING`, the file names are ordered as if they are version numbers - (we use `packaging.version.LegacyVersion()` to preform the sort) -- the whole file-name is treated as a version, so "aaaa-1.0.0.yaml" is sorted before "bbbb-9.0.0.yaml" - (take care not to change your file-name prefixes, or alternatively don't include a prefix and use "1.0.0.ymal") +- no files are matched, as `max_recursion_depth` is `0`, and there is no component at the root ``` ## Uninstall the connector @@ -191,7 +270,7 @@ Notes: 1. Remove all Artifactory catalog entries from the `Manage Components` panel 2. Stop JupyterLab 3. Uninstall the `elyra-artifactory-catalog-connector` package: - ``` + ```bash $ pip uninstall elyra-artifactory-catalog-connector ``` @@ -199,5 +278,3 @@ Notes: - __Problem:__ The palette does not display any components from the configured catalog. - __Solution:__ If the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. -- __Problem:__ The pallet does not reflect the current state of the Artifactory repo. - - __Solution:__ Trigger the catalog to refresh by editing the catalog, and clicking "SAVE & CLOSE" (without making any changes). From a88cfe9feaab0fe9743e0dc03f4a318da8801e3e Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Fri, 3 Jun 2022 15:30:00 +1000 Subject: [PATCH 07/10] update catalog metadata for elyra 3.9 --- .../artifactory-catalog.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json index 838bc9f..b4194e5 100644 --- a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json @@ -66,7 +66,8 @@ "format": "uri", "uihints": { "category": "Source", - "placeholder": "https://example.org/artifactory/" + "ui:placeholder": "https://example.com/artifactory/", + "placeholder": "https://example.com/artifactory/" } }, "artifactory_username": { @@ -82,8 +83,9 @@ "description": "Password or API Key for the Artifactory server", "type": "string", "uihints": { - "secure": true, - "category": "Source" + "category": "Source", + "ui:field": "password", + "secure": true } }, "repository_name": { @@ -106,7 +108,7 @@ }, "max_recursion_depth": { "title": "Maximum Recursion Depth", - "description": "Maximum folder depth to recurse looking for '__COMPONENT__' files", + "description": "Maximum folder depth to recurse looking for '__COMPONENT__' marker files", "type": "string", "pattern": "^[0-9]+$", "default": "0", @@ -126,7 +128,7 @@ }, "file_filter": { "title": "File Filter", - "description": "Unix-like file name filter. ('*' match everything; '?' any single character; '[seq]' character in seq; '[!seq]' character not in seq, '[0-9]' any number)", + "description": "Fnmatch file name filter: '*' match everything; '?' any single character; '[seq]' character in seq; '[!seq]' character not in seq; '[0-9]' any single number;", "default": "*.yaml", "type": "string", "uihints": { @@ -135,7 +137,7 @@ }, "file_ordering": { "title": "File Ordering", - "description": "Order in which files are processed per folder", + "description": "Order in which files are processed per folder. 'NAME' alphanumeric; 'VERSION' packaging.version.LegacyVersion();", "type": "string", "enum": [ "NAME_ASCENDING", From b97b758ec6867f83c646bb560d12cfb80f7ee7bb Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Mon, 18 Jul 2022 15:10:10 +1000 Subject: [PATCH 08/10] fix issues from review --- .../artifactory-connector/README.md | 7 ++++++- .../artifactory_catalog_connector/artifactory-catalog.json | 6 +++--- .../artifactory-connector/setup.py | 4 ++-- component-catalog-connectors/connector-directory.md | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/README.md b/component-catalog-connectors/artifactory-connector/README.md index 6c54b6d..cf41142 100644 --- a/component-catalog-connectors/artifactory-connector/README.md +++ b/component-catalog-connectors/artifactory-connector/README.md @@ -1,6 +1,11 @@ ## Artifactory component catalog connector -This catalog connector enables Elyra to load pipelines components from a generic-type Artifactory repo. +This catalog connector enables Elyra to load Kubeflow Pipelines components from a generic-type Artifactory repo. + +> 🟨 __Note__ 🟨 +> +> Currently, this connector only works with `--runtime_type=KUBEFLOW_PIPELINES`, +> we welcome contributions for other runtime types. ## Install the connector diff --git a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json index b4194e5..e8d3add 100644 --- a/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json +++ b/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json @@ -1,6 +1,6 @@ { - "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", - "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json", + "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/main/elyra/metadata/schemas/meta-schema.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/main/component-catalog-connectors/artifactory-connector/artifactory_catalog_connector/artifactory-catalog.json", "title": "Artifactory Component Catalog", "name": "artifactory-catalog", "schemaspace": "component-catalogs", @@ -9,7 +9,7 @@ "uihints": { "title": "Artifactory Component Catalog", "icon": "", - "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/artifactory-connector" + "reference_url": "https://github.com/elyra-ai/examples/tree/main/component-catalog-connectors/artifactory-connector" }, "properties": { "schema_name": { diff --git a/component-catalog-connectors/artifactory-connector/setup.py b/component-catalog-connectors/artifactory-connector/setup.py index e747acd..5f7ff08 100644 --- a/component-catalog-connectors/artifactory-connector/setup.py +++ b/component-catalog-connectors/artifactory-connector/setup.py @@ -30,7 +30,7 @@ setup_args = dict( name="elyra-artifactory-catalog-connector", version=version_ns["__version__"], - url="https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/artifactory-connector", + url="https://github.com/elyra-ai/examples/tree/main/component-catalog-connectors/artifactory-connector", description="Elyra component catalog connector for Artifactory", long_description=long_desc, author="Elyra Maintainers", @@ -58,10 +58,10 @@ "Topic :: Software Development", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], ) diff --git a/component-catalog-connectors/connector-directory.md b/component-catalog-connectors/connector-directory.md index dc64e7d..4a20ed9 100644 --- a/component-catalog-connectors/connector-directory.md +++ b/component-catalog-connectors/connector-directory.md @@ -9,6 +9,6 @@ The following catalog connectors should work with Elyra version 3.3 and above. C | --- | --- | | [Apache Airflow example catalog](airflow-example-components-connector) | Provides access to a small set of curated Apache Airflow operators that you can use to get started with the Visual Pipeline Editor. | | [Kubeflow Pipelines example catalog](kfp-example-components-connector) | Provides access to a small set of curated Kubeflow Pipelines components that you can use to get started with the Visual Pipeline Editor. | -| [Artifactory](artifactory-connector) | Enables Elyra to load pipelines components from a generic-type Artifactory repo. | +| [Artifactory](artifactory-connector) | Enables Elyra to load Kubeflow Pipelines components components from a generic-type Artifactory repo. | | [Machine Learning Exchange](mlx-connector/) | This LFAI project provides an open source Data and AI assets catalog and execution engine for Kubeflow Pipelines. | From 56c96dbc36e70488a6569ec34f15ec9c30904baf Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Wed, 20 Jul 2022 10:43:01 -0700 Subject: [PATCH 09/10] Improve markdown rendering --- .../artifactory-connector/README.md | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/component-catalog-connectors/artifactory-connector/README.md b/component-catalog-connectors/artifactory-connector/README.md index cf41142..59e22f1 100644 --- a/component-catalog-connectors/artifactory-connector/README.md +++ b/component-catalog-connectors/artifactory-connector/README.md @@ -193,9 +193,9 @@ component_group/ ### Example 1 +**Connector configuration:** + ``` -Configs: -========= artifactory_url = https://example.com/artifactory/ repository_name = elyra-components repository_path = / @@ -203,27 +203,30 @@ max_recursion_depth = 3 max_files_per_folder = -1 file_filter = *.yaml file_ordering = VERSION_DESCENDING +``` + +**Matched components:** -Matched: -========= +``` /component_one/1.0.0.yaml /component_one/2.0.0.yaml /component_group/component_two/1.0.0.yaml /component_group/component_two/2.0.0.yaml /component_group/component_three/1.0.0.yaml /component_group/component_three/2.0.0.yaml +``` + +**Notes:** -Notes: -========= - the `/missing_marker` files are not matched as this folder does not contain a `__COMPONENT__` marker - the `/component_group/component_two/nested_component` files are not matched as recursion stops at the first `__COMPONENT__` marker -``` + ### Example 2 +**Connector configuration:** + ``` -Configs: -========= artifactory_url = https://example.com/artifactory/ repository_name = elyra-components repository_path = / @@ -231,28 +234,30 @@ max_recursion_depth = 3 max_files_per_folder = 1 file_filter = *.yaml file_ordering = VERSION_DESCENDING +``` + +**Matched components:** -Matched: -========= +``` /component_one/2.0.0.yaml /component_group/component_two/2.0.0.yaml /component_group/component_three/2.0.0.yaml +``` + +**Notes:** -Notes: -========= - as `max_files_per_folder` is `1`, only ONE file from each folder is matched - as `file_ordering` is `VERSION_DESCENDING`, the HIGHEST version file is returned - we use `packaging.version.LegacyVersion()` to preform the sort - the `file_ordering` is applied separately within each folder - the WHOLE file-name is treated as a version, so "aaaa-1.0.0.yaml" would be sorted before "bbbb-9.0.0.yaml" (to avoid this problem, don't include any prefix on file names, for example `1.0.0.yaml`, `1.1.0.yaml`) -``` ### Example 3 +**Connector configuration:** + ``` -Configs: -========= artifactory_url = https://example.com/artifactory/ repository_name = elyra-components repository_path = / @@ -260,15 +265,11 @@ max_recursion_depth = 0 max_files_per_folder = -1 file_filter = *.yaml file_ordering = VERSION_DESCENDING +``` -Matched: -========= -NONE +**Matched components:** -Notes: -========= -- no files are matched, as `max_recursion_depth` is `0`, and there is no component at the root -``` +None. No files are matched, as `max_recursion_depth` is `0`, and there is no component at the root. ## Uninstall the connector @@ -282,4 +283,4 @@ Notes: ## Troubleshooting - __Problem:__ The palette does not display any components from the configured catalog. - - __Solution:__ If the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. +- __Solution:__ If the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. From 6a31fb753c0aa25d06a95c8cfd2ebb871a096c18 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Wed, 20 Jul 2022 10:55:29 -0700 Subject: [PATCH 10/10] Fix typo --- component-catalog-connectors/connector-directory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component-catalog-connectors/connector-directory.md b/component-catalog-connectors/connector-directory.md index 4a20ed9..47b63b1 100644 --- a/component-catalog-connectors/connector-directory.md +++ b/component-catalog-connectors/connector-directory.md @@ -9,6 +9,6 @@ The following catalog connectors should work with Elyra version 3.3 and above. C | --- | --- | | [Apache Airflow example catalog](airflow-example-components-connector) | Provides access to a small set of curated Apache Airflow operators that you can use to get started with the Visual Pipeline Editor. | | [Kubeflow Pipelines example catalog](kfp-example-components-connector) | Provides access to a small set of curated Kubeflow Pipelines components that you can use to get started with the Visual Pipeline Editor. | -| [Artifactory](artifactory-connector) | Enables Elyra to load Kubeflow Pipelines components components from a generic-type Artifactory repo. | +| [Artifactory](artifactory-connector) | Enables Elyra to load Kubeflow Pipelines components from a generic-type Artifactory repo. | | [Machine Learning Exchange](mlx-connector/) | This LFAI project provides an open source Data and AI assets catalog and execution engine for Kubeflow Pipelines. |