0.2.6

.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install poetry
+        poetry install --no-root
+    - name: Compile protobufs
+      run: |
+        poetry run python utils/download_protobufs.py
+    - name: Build and publish
+      env:
+        PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        poetry build
+        poetry publish --username $PYPI_USERNAME --password $PYPI_PASSWORD

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Run python tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install poetry
+        poetry install --no-root
+    - name: Compile Protobufs
+      run: |
+        poetry run python utils/download_protobufs.py
+        poetry install
+    - name: Test with pytest
+      run: |
+        poetry run pytest --spec

CHANGELOG.md

@@ -0,0 +1,6 @@
+0.2.6 - 23 July 2021
+--------------------
+* Added prometheus to protoc script
+* Made protoc script compile all protos at the same time, since this appears to work now
+* Added github actions workflows
+* Removed travis-ci workflows

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "envoy_data_plane"
-version = "0.2.5"
+version = "0.2.6"
 description = "Python dataclasses for the Envoy Data-Plane-API"
 authors = ["Vasili Syrakis <vsyrakis@atlassian.com>"]
 license = "MIT"

utils/download_protobufs.py

@@ -11,123 +11,108 @@
 structlog.configure()
 logger = structlog.get_logger()
 
-build_directory = Path('BUILD')
+build_directory = Path("BUILD")
 if not build_directory.exists():
-    logger.msg('Creating BUILD directory')
+    logger.msg("Creating BUILD directory")
     build_directory.mkdir()
 os.chdir(build_directory)
 
-utf8 = 'utf-8'
+utf8 = "utf-8"
 
-proto_include = protoc.pkg_resources.resource_filename('grpc_tools', '_proto')
-output = Path('../src/envoy_data_plane')
-envoy = Path('./envoy')
-envoy_api = Path('./envoy/api')
-envoy_api_v2 = Path('./envoy/api/v2')
-proto_args = [
-    __file__,
-    f'--proto_path=.',
-    f'--proto_path={envoy}',
-    f'--proto_path={envoy_api}',
-    f'--proto_path={envoy_api_v2}',
-    f'--proto_path={proto_include}',
-]
+ENVOY_VERSION = "1.16.2"
 
-Package = namedtuple('Package', ['url', 'name', 'directory'])
+proto_include = protoc.pkg_resources.resource_filename("grpc_tools", "_proto")
+envoy = Path("./envoy")
+envoy_api = Path("./envoy/api")
+envoy_api_v2 = Path("./envoy/api/v2")
+
+Package = namedtuple("Package", ["url", "name", "directory", "namespace"])
 packages = {
     Package(
-        url='https://github.com/envoyproxy/envoy/archive/refs/tags/v1.16.2.zip',
-        name='envoy',
-        directory='envoy-1.16.2/api'
+        url=f"https://github.com/envoyproxy/envoy/archive/refs/tags/v{ENVOY_VERSION}.zip",
+        name="envoy",
+        namespace="envoy",
+        directory=f"envoy-{ENVOY_VERSION}/api",
+    ),
+    Package(
+        url="https://github.com/googleapis/googleapis/archive/master.zip",
+        name="google",
+        namespace="google",
+        directory="googleapis-master",
     ),
     Package(
-        url='https://github.com/googleapis/googleapis/archive/master.zip',
-        name='google',
-        directory='googleapis-master'
+        url="https://github.com/cncf/udpa/archive/refs/tags/v0.0.1.zip",
+        name="udpa",
+        namespace="udpa",
+        directory="udpa-0.0.1",
     ),
     Package(
-        url='https://github.com/cncf/udpa/archive/refs/tags/v0.0.1.zip',
-        name='udpa',
-        directory='udpa-0.0.1'
+        url="https://github.com/envoyproxy/protoc-gen-validate/archive/main.zip",
+        name="validate",
+        namespace="validate",
+        directory="protoc-gen-validate-main",
     ),
     Package(
-        url='https://github.com/envoyproxy/protoc-gen-validate/archive/main.zip',
-        name='validate',
-        directory='protoc-gen-validate-main'
+        url="https://github.com/census-instrumentation/opencensus-proto/archive/refs/tags/v0.2.0.zip",
+        name="opencensus",
+        namespace="opencensus",
+        directory="opencensus-proto-0.2.0/src",
     ),
     Package(
-        url='https://github.com/census-instrumentation/opencensus-proto/archive/refs/tags/v0.2.0.zip',
-        name='opencensus',
-        directory='opencensus-proto-0.2.0/src'
-    )
+        url="https://github.com/prometheus/client_model/archive/refs/tags/v0.2.0.zip",
+        name="prometheus",
+        namespace=".",
+        directory="client_model-0.2.0",
+    ),
 }
 
 
 for package in packages:
-    namespace = Path(package.name)
+    name = Path(package.name)
+    namespace = Path(package.namespace)
     proto_root = Path(package.directory)
-    archive = Path(f'{package.name}.zip')
+    archive = Path(f"{package.name}.zip")
 
     if not archive.exists():
-        logger.msg(f'Downloading {package.name} protocol buffers from Github')
-        with open(archive, 'wb+') as zf:
+        logger.msg(f"Downloading {package.name} protocol buffers from Github")
+        with open(archive, "wb+") as zf:
             zf.write(requests.get(package.url).content)
 
-    if namespace.exists():
-        logger.msg(f'{namespace.absolute()} exists')
+    if name.exists():
+        logger.msg(f"{name.absolute()} exists")
     else:
-        logger.msg(f'Extracting {namespace} archive')
-        with zipfile.ZipFile(archive, 'r') as zipref:
+        logger.msg(f"Extracting {name} archive")
+        with zipfile.ZipFile(archive, "r") as zipref:
             protos = [
-                member
-                for member in zipref.namelist()
-                if member.endswith('.proto')
+                member for member in zipref.namelist() if member.endswith(".proto")
             ]
             for file in protos:
-                logger.msg(f'Extracting {file}')
-                zipref.extract(
-                    member=file,
-                    path='.'
-                )
-        shutil.copytree(
-            proto_root/namespace,
-            namespace
-        )
-
-
-def everything():
-    proto_files = [
-        str(file) for file in envoy.glob('**/*')
-        if file.suffix == '.proto'
-    ]
-    proto_paths = [
-        f'--proto_path={folder}'
-        for folder in Path('.').glob('**/*')
-        if folder.is_dir()
-    ]
-    args = deepcopy(proto_args)
-    args += proto_paths
-    args += [f'--python_betterproto_out={output}']
-    logger.msg('Running GRPC tools to generate python code from protocol buffers')
-    files = list()
-    for f in proto_files:
-        files.append(f)
-        if len(files) >= 100:
-            protoc.main(args + files)
-            files.clear()
-    protoc.main(args + files)
-
-
-def xds():
+                logger.msg(f"Extracting {file}")
+                zipref.extract(member=file, path=".")
+        shutil.copytree(proto_root / namespace, name)
+
+
+output = Path("../src/envoy_data_plane")
+proto_args = [
+    __file__,
+    f"--proto_path=.",
+    f"--proto_path={envoy}",
+    f"--proto_path={envoy_api}",
+    f"--proto_path={envoy_api_v2}",
+    f"--proto_path={proto_include}",
+]
+
+
+def compile_all():
+    logger.msg("Running GRPC tools to generate python code from protocol buffers")
+    proto_files = [str(file) for file in envoy.glob("**/*") if file.suffix == ".proto"]
     proto_paths = [
-        str(envoy_api_v2.joinpath(Path(f'{discovery_type}.proto')))
-        for discovery_type in ['cds', 'lds', 'rds', 'eds', 'srds']
+        f"--proto_path={folder}" for folder in Path(".").glob("**/*") if folder.is_dir()
     ]
     args = deepcopy(proto_args)
     args += proto_paths
-    args += [f'--python_betterproto_out={output}']
-    protoc.main(args)
+    args += [f"--python_betterproto_out={output}"]
+    protoc.main((*args, *proto_files))
 
 
-everything()
-xds()
+compile_all()