Revert "Delete all Mesos-using code (+ misc. cleanups) (#215)" (#216)

This reverts commit f826a14.

Makefile

@@ -5,27 +5,24 @@ else
 	BUILD_ENV?=$(shell hostname -f)
 endif
 
-.PHONY: venv
 venv:
 	tox -e venv
 
-.PHONY: test
 test:
 	tox
 
-.PHONY: tox_%
 tox_%:
 	tox -e $*
 
-.PHONY: docs
+itest:
+	tox -e integration
+
 docs:
 	tox -e docs
 
-.PHONY: pypi
 pypi:
 	tox -e pypi
 
-.PHONY: clean
 clean:
 	rm -rf docs/build
 	find . -name '*.pyc' -delete

README.md

@@ -6,9 +6,47 @@ Interfaces and shared infrastructure for generic task processing (also known as
 
 ### Pre-requisites
 
++ [Docker](https://www.docker.com/get-docker)
 + [Python 3.8](https://www.python.org/downloads/)
 + [Virtualenv](https://virtualenv.pypa.io/en/stable/installation/)
 
+### Running examples
+
+[hello-world.py](/examples/hello-world/py) is a very simple annotated example that launches a task to echo `hello world`. From the root of the repository, run:
+
+    docker-compose -f examples/cluster/docker-compose.yaml \
+      run playground examples/hello-world.py
+
+This will bring up a single master, single agent Mesos cluster using [Docker Compose](https://docs.docker.com/compose/) and launch a single task which will print "hello world" to the sandbox's stdout before terminating.
+
+Other examples available include:
++ async.py
+Example of the [async](#async) task runner.
+
++ dynamo_persistence.py
+Example that shows how task events may be persisted to [DynamoDB](https://aws.amazon.com/dynamodb) using the `stateful` plugin.
+
++ file_persistence.py
+Example that shows how task events may be persisted to disk using the `stateful` plugin.
+
++ promise.py
+Example that shows how the [promise/future](#Promise/Future) task runner (not yet implemented) may be used.
+
++ subscription.py
+Example of the [subscription](#subscription) task runner.
+
++ sync.py
+Brief example using the [sync](#sync) task runner.
+
++ timeout.py
+Example that shows how to timeout a task execution using the `timeout` plugin.
+
++ retry.py
+Example that shows how to retry a task on failure using the `retry` plugin.
+
++ task_logging.py
+Example that shows how to fetch task logs from Mesos agents using the `logging` plugin.
+
 ### Running tests
 
 From the root of the repository, run:
@@ -27,10 +65,30 @@ From the root of the repository, run:
 
 ### /plugins
 
-Plugins can be chained to create a task execution pipeline with more than one property.
+Plugins can be chained to create a task execution pipeline with more than one property. Please refer to persistence/retry/timeout examples.
+
+#### mesos
+Implements all required interfaces to talk to Mesos deployment. This plugin uses [PyMesos](https://github.com/douban/pymesos) to communicate with Mesos.
+
+#### timeout
+Implements an executor to timeout task execution.
+
+#### retrying
+Implements an executor to retry task execution upon failure.
+
+#### logging
+Implements an executor to retrieve task logs from Mesos agents. Note that it has to be the immediate upstream executor of the mesos executor.
+
+##### Configuration options
+
+- authentication\_principal Mesos principal
+- credential\_secret\_file path to file containing Mesos secret
+- mesos\_address host:port to connect to Mesos cluster
+- event_translator a fucntion that maps Mesos-specific events to `Event` objects
+
+#### stateful
 
-#### Kubernetes
-Implements all required interfaces to talk to Kubernetes. This plugin uses [kubernetes-client](https://github.com/kubernetes-client/python) to communicate with Kubernetes.
+TODO: documentation
 
 ### /runners
 

docs/source/generated/task_processing.interfaces.event.rst

@@ -1,7 +1,7 @@
-task\_processing.interfaces.event module
-========================================
+task\_processing\.interfaces\.event module
+==========================================
 
 .. automodule:: task_processing.interfaces.event
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.interfaces.rst

@@ -1,21 +1,19 @@
-task\_processing.interfaces package
-===================================
+task\_processing\.interfaces package
+====================================
 
 Submodules
 ----------
 
 .. toctree::
-   :maxdepth: 4
 
    task_processing.interfaces.event
-   task_processing.interfaces.persistence
    task_processing.interfaces.runner
    task_processing.interfaces.task_executor
 
 Module contents
 ---------------
 
 .. automodule:: task_processing.interfaces
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.interfaces.runner.rst

@@ -1,7 +1,7 @@
-task\_processing.interfaces.runner module
-=========================================
+task\_processing\.interfaces\.runner module
+===========================================
 
 .. automodule:: task_processing.interfaces.runner
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.interfaces.task_executor.rst

@@ -1,7 +1,7 @@
-task\_processing.interfaces.task\_executor module
-=================================================
+task\_processing\.interfaces\.task\_executor module
+===================================================
 
 .. automodule:: task_processing.interfaces.task_executor
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.plugins.mesos.execution_framework.rst

@@ -0,0 +1,7 @@
+task\_processing\.plugins\.mesos\.execution\_framework module
+=============================================================
+
+.. automodule:: task_processing.plugins.mesos.execution_framework
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.plugins.mesos.mesos_executor.rst

@@ -0,0 +1,7 @@
+task\_processing\.plugins\.mesos\.mesos\_executor module
+========================================================
+
+.. automodule:: task_processing.plugins.mesos.mesos_executor
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.plugins.mesos.rst

@@ -0,0 +1,19 @@
+task\_processing\.plugins\.mesos package
+========================================
+
+Submodules
+----------
+
+.. toctree::
+
+   task_processing.plugins.mesos.execution_framework
+   task_processing.plugins.mesos.mesos_executor
+   task_processing.plugins.mesos.translator
+
+Module contents
+---------------
+
+.. automodule:: task_processing.plugins.mesos
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.plugins.mesos.translator.rst

@@ -0,0 +1,7 @@
+task\_processing\.plugins\.mesos\.translator module
+===================================================
+
+.. automodule:: task_processing.plugins.mesos.translator
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.plugins.rst

@@ -1,18 +1,17 @@
-task\_processing.plugins package
-================================
+task\_processing\.plugins package
+=================================
 
 Subpackages
 -----------
 
 .. toctree::
-   :maxdepth: 4
 
-   task_processing.plugins.kubernetes
+    task_processing.plugins.mesos
 
 Module contents
 ---------------
 
 .. automodule:: task_processing.plugins
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.rst

@@ -5,26 +5,15 @@ Subpackages
 -----------
 
 .. toctree::
-   :maxdepth: 4
 
-   task_processing.interfaces
-   task_processing.plugins
-   task_processing.runners
-
-Submodules
-----------
-
-.. toctree::
-   :maxdepth: 4
-
-   task_processing.metrics
-   task_processing.task_processor
-   task_processing.utils
+    task_processing.interfaces
+    task_processing.plugins
+    task_processing.runners
 
 Module contents
 ---------------
 
 .. automodule:: task_processing
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.runners.async.rst

@@ -0,0 +1,7 @@
+task\_processing\.runners\.async module
+=======================================
+
+.. automodule:: task_processing.runners.async
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/generated/task_processing.runners.promise.rst

@@ -1,7 +1,7 @@
-task\_processing.runners.promise module
-=======================================
+task\_processing\.runners\.promise module
+=========================================
 
 .. automodule:: task_processing.runners.promise
-   :members:
-   :undoc-members:
-   :show-inheritance:
+    :members:
+    :undoc-members:
+    :show-inheritance: