ODI-3611 Add cache manager client

- Remove conda artifacts from Makefile
- Add CacheManagerClient with simple interface to get/set data
- Add CacheManagerUtils with helper function to handle get/set
- Add examples/check_disk.py, plugin written using cachemanager

Makefile

@@ -6,7 +6,7 @@
 # ------------------------------------------------------------------------------
 
 # List special make targets that are not associated with files
-.PHONY: help version conda conda_dev build wheel vtest test lint doc format clean
+.PHONY: help venv version wheel test lint doc format clean
 
 # Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
 SHELL=/bin/bash
@@ -32,16 +32,6 @@ PKGNAME=${VENDOR}-${PROJECT}
 # Current directory
 CURRENTDIR=$(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 
-# Conda environment
-CONDA_ENV=$(shell dirname ${CURRENTDIR})/env-${PROJECT}
-
-# Include default build configuration
-include $(CURRENTDIR)/config.mk
-
-# extract all packages
-ALLPACKAGES=$(shell cat conda/meta.yaml | grep -oP '^\s*-\s\K(.*)' | sed "s/.*${PROJECT}//" | sed '/^\s*$$/d' | sort -u | tr -d ' ' | sed 's/[^ ][^ ]*/"&"/g' | tr '\r\n' ' ')
-
-
 # --- MAKE TARGETS ---
 
 # Display general help about this command
@@ -50,12 +40,9 @@ help:
 	@echo "$(PROJECT) Makefile."
 	@echo "The following commands are available:"
 	@echo ""
+	@echo "    make venv       : Set up development virtual environment"
 	@echo "    make version    : Set version from VERSION file"
-	@echo "    make conda      : Build minimal Conda environment"
-	@echo "    make conda_dev  : Build development Conda environment"
-	@echo "    make build      : Build a Conda package"
 	@echo "    make wheel      : Build a Wheel package"
-	@echo "    make vtest      : Execute tests inside a Python 2.7 virtualenv"
 	@echo "    make test       : Execute test command"
 	@echo "    make lint       : Evaluate code"
 	@echo "    make doc        : Start a server to display source code documentation"
@@ -65,44 +52,35 @@ help:
 
 all: help
 
+# Build dev env
+venv: venv/bin/activate
+
+venv/bin/activate: requirements.txt
+	test -d venv || virtualenv venv
+	source venv/bin/activate ; \
+	pip install -r requirements.txt ; \
+	pip install -e '.[test]'
+
 # Set the version from VERSION file
 version:
-	sed -i "s/version:.*$$/version: $(VERSION)/" conda/meta.yaml
 	sed -i "s/__version__.*$$/__version__ = '$(VERSION)'/" plugnpy/__init__.py
 
-# Build minimal Conda environment
-conda:
-	./conda/setup-conda.sh
-
-# Build development Conda environment
-conda_dev:
-	ENV_NAME=env-dev-plugnpy ./conda/setup-conda.sh
-	. ../env-dev-plugnpy/bin/activate && \
-	../env-dev-plugnpy/bin/conda install --override-channels $(CONDA_CHANNELS) -y $(ALLPACKAGES)
-
-# Build a conda package
-build: clean version conda
-	mkdir -p target
-	PROJECT_ROOT=${CURRENTDIR} "${CONDA_ENV}/bin/conda" build --prefix-length 128 --no-anaconda-upload --override-channels $(CONDA_CHANNELS) conda
-
 # Build a Wheel package
-wheel: clean version
+wheel: clean version venv
+	source venv/bin/activate ; \
 	python setup.py sdist bdist_wheel
 
-# Test the project in a Python 2.7 virtual environment
-vtest:
-	rm -rf venv
-	virtualenv -p /usr/bin/python2.7 venv
-	source venv/bin/activate && pip install -e .[test] && make test && coverage html
-
 # Test using setuptools
-test:
-	python setup.py test
+test: venv
+	source venv/bin/activate ; \
+	python setup.py test ; \
+	coverage html
 
 # Evaluate code
-lint:
-	pyflakes ${PROJECT}
-	pylint ${PROJECT}
+lint: venv
+	source venv/bin/activate ; \
+	pyflakes ${PROJECT} ; \
+	pylint ${PROJECT} ; \
 	pycodestyle --max-line-length=120 ${PROJECT}
 
 # Generate source code documentation

README.md

@@ -7,7 +7,7 @@
 [![Master Coverage Status](https://coveralls.io/repos/opsview/plugnpy/badge.svg?branch=master&service=github)](https://coveralls.io/github/opsview/plugnpy?branch=master)
 
 * **category**    Libraries
-* **copyright**   2018-2019 Opsview Ltd
+* **copyright**   2003-2019 Opsview Ltd
 * **license**     Apache License Version 2.0 (see [LICENSE](LICENSE))
 * **link**        https://github.com/opsview/plugnpy
 
@@ -37,44 +37,27 @@ To see all available options:
 make help
 ```
 
-To build a Conda development environment:
+To create a development environment:
 ```
-make conda_dev
-. activate
+make venv
 ```
 
-To test inside a `conda_dev` environment using setuptools:
+To run the tests in the development environment:
 ```
 make test
 ```
 
-To build and test the project inside a Conda environment:
-```
-make build
-```
-
-The coverage report is available at:
-```
-env-~#PROJECT#~/conda-bld/coverage/htmlcov/index.html
-```
-
-
-## Building the Library
-
-Should you wish (or if you've made improvements to the source code and you want to use them), the library can be built with the following command.
-
-```
-python setup.py sdist
-```
-
-or
-
+To build the library in the development environment:
 ```
 make wheel
 ```
 
 This will create a `plugnpy-VERSION.tar.gz` file in the `dist` directory which can be installed in the same way as the prepackaged one above.
 
+The coverage report is available at:
+```
+htmlcov/index.html
+```
 
 ## Writing Checks
 
@@ -286,3 +269,97 @@ To use the parser, create an object of type **plugnpy.Parser** and use as you wo
 | AssumedOK              | To be thrown when the status of the check cannot be identified. This is usually used when the check requires the result of a previous run and this is the first run. |
 | InvalidMetricThreshold | This shouldn't be thrown in a plugin. It is used internally in checks.py when an invalid metric threshold is passed in.                                              |
 | InvalidMetricName      | This shouldn't be thrown in a plugin. It is used internally in checks.py when an invalid metric name is passed in.                                              |
+
+## Cache Manager client
+
+**plugnpy** comes with an http client which is able to connect to the opsview-cachemanager component.
+This allows the plugins to use the cache manager to store temporary data into memory which can be consumed by other
+servicechecks which require the same data.
+
+The module consists of two classes, namely **CacheManagerClient** and **CacheManagerUtils**, which provide easy to use
+interfaces to communicate with the opsview-cachemanager.
+
+### CacheManagerClient
+A simple client to set or get cached data from the cache manager.
+
+The cache manager client requires the **namespace** of the plugin and the **host** ip and **port** of the cache manager
+to be supplied. These are provided to the plugin as opsview-executor encrypted environment variables.
+
+```python
+host = os.environ.get('OPSVIEW_CACHE_MANAGER_HOST')
+port = os.environ.get('OPSVIEW_CACHE_MANAGER_PORT')
+namespace = os.environ.get('OPSVIEW_CACHE_MANAGER_NAMESPACE')
+```
+
+A cache manager client can then be created:
+```python
+client = CacheManagerClient(host, port, namespace)
+```
+
+Items inserted into the cache manager are namespaced, ensuring naming collisions are avoided and potentially sensitive
+data cannot be read by other unauthorized plugins.
+
+Optionally, when creating the client, the **concurrency**, **connection_timeout** and **network_timeout** parameters can
+be specified to modify the number of concurrent http connection allowed (default: 1), the number of seconds before the
+HTTP connection times out (default: 30) and the number of seconds before the data read times out (default: 30),
+respectively.
+
+```python
+client = CacheManagerClient(host, port, namespace, concurrency=1, connection_timeout=30,
+                            network_timeout=30)
+```
+Once a cache manager client has been created, the **get_data** and **set_data** methods can be used to get and set data
+respectively.
+
+The **set_data** method can be called with the **key** and **data** parameters, this will store the specified data,
+under the given key. Optionally, the **ttl** parameter can be used to specify the number of seconds the data is valid
+for (default: 900). It is expected that session information and other temporary data will be stored in the cache
+manager. 15 minutes has been chosen as the default to ensure data does not have to be recreated too often, but in the
+event of a change in data, the cached information does not persist for too long.
+
+```python
+client.set_data(key, data, ttl=900)
+```
+
+The **get_data** method can be called with the **key** parameter to retrieve data stored under the specified key.
+Optionally, the **max_wait_time** parameter can be used to specify the number of seconds to wait before timing out
+(default: 30).
+
+```python
+client.get_data(key, max_wait_time=30)
+```
+
+Calling the **get_data** method when the data exists in the cache manager will return the data. However, if the data
+does not exist in the cache manager, it will return a lock. Obtaining a lock means the cache manager expects the
+component to make the call to get the data directly and then use the **set_data** method to set the data in the cache
+manager, ready to be used by other components. Any concurrent components calling the **get_data** method will block if
+they cannot obtain the lock, this ensures that only one component sets the data. Once the data has been set, all blocked
+components will be unblocked and return the newly cached data. The **max_wait_time** parameter of the **get_data**
+method has a default of 30 seconds, but needs to be large enough for this cycle to be completed.
+
+
+### CacheManagerUtils
+
+To simplify calls to the cache manager, **plugnpy** provides a helper utility method **get_via_cachemanager**, this will
+create the cache manager client and call the **get_data** and **set_data** methods as required.
+
+This method expects five parameters:
+* no_cachemanager: True if cache manager is not required, False otherwise.
+* key: The key to store the data under.
+* func: The function to retrieve the data, if the data is not in the cache manager.
+* args: The arguments to pass to the user's data retrieval function.
+* kwargs: The keyword arguments to pass to the user's data retrieval function.
+
+
+```python
+def api_call(string):
+  return string[::-1]
+
+CacheManagerUtils.get_via_cachemanager(no_cachemanager, 'my_key', api_call, 'hello')
+```
+
+In this example, if the data exists in the cache manager under the key `'my_key'`, the call to **get_via_cachemanager**
+will simply return the data. However, if the data does not exist in the cache manager, the call to
+**get_via_cachemanager** will call the **api_call** method with the argument `'hello'` and then set the data in the
+cachemanager, so future calls can use the data from the cache manager.
+

VERSION

@@ -1 +1 @@
-2.0.1
+2.0.2

examples/check_cpu.py

@@ -1,3 +1,18 @@
+#!/opt/opsview/python/bin/python
+# Copyright (C) 2003-2019 Opsview Limited. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # check_cpu.py - An example Opsview plugin written with plugnpy
 from __future__ import print_function