From f62a28a996cd98ca97bc622dc567186d625afd2e Mon Sep 17 00:00:00 2001 From: brendan Date: Tue, 19 Nov 2019 10:31:04 +0000 Subject: [PATCH] Removed SSH reloading of Jenkins, since is possible by REST call instead now (since v1.8.0 of Helm chart). Timestamped & categorised Python logs instead of print statements. Reload validation boolean output to text file. Created Jenkins user and permissioned files to it, rather than root. --- Dockerfile | 5 ++++ sidecar/helpers.py | 17 ++++++++---- sidecar/resources.py | 65 ++++++++++++++++++++++---------------------- sidecar/sidecar.py | 40 +++++++++++++++++---------- 4 files changed, 75 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03c33d1b..6aff11fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,11 @@ FROM python:3.7-slim WORKDIR /app COPY requirements.txt . RUN pip install -r requirements.txt +RUN groupadd -r -g 1000 jenkins && useradd -r -u 1000 -g jenkins jenkins +USER root +RUN chown -R jenkins:jenkins /app COPY sidecar/* ./ +RUN chmod 700 /app ENV PYTHONUNBUFFERED=1 +USER jenkins CMD [ "python", "-u", "/app/sidecar.py" ] diff --git a/sidecar/helpers.py b/sidecar/helpers.py index 650d6637..5e3364f2 100644 --- a/sidecar/helpers.py +++ b/sidecar/helpers.py @@ -19,15 +19,15 @@ def writeTextToFile(folder, filename, data): f.close() -def removeFile(folder, filename): +def removeFile(folder, filename, logger): completeFile = folder + "/" + filename if os.path.isfile(completeFile): os.remove(completeFile) else: - print(f"Error: {completeFile} file not found") + logger.error(f"Error: %s file not found" % completeFile) -def request(url, method, payload=None): +def request(url, method, logger, payload=None): retryTotal = 5 if os.getenv('REQ_RETRY_TOTAL') is None else int(os.getenv('REQ_RETRY_TOTAL')) retryConnect = 5 if os.getenv('REQ_RETRY_CONNECT') is None else int( os.getenv('REQ_RETRY_CONNECT')) @@ -45,7 +45,7 @@ def request(url, method, payload=None): r.mount('http://', HTTPAdapter(max_retries=retries)) r.mount('https://', HTTPAdapter(max_retries=retries)) if url is None: - print("No url provided. Doing nothing.") + logger.info("No url provided. Doing nothing.") return # If method is not provided use GET as default @@ -53,5 +53,12 @@ def request(url, method, payload=None): res = r.get("%s" % url, timeout=timeout) elif method == "POST": res = r.post("%s" % url, json=payload, timeout=timeout) - print(f"{method} request sent to {url}. Response: {res.status_code} {res.reason}") + if res.status_code == 200: + logger.info("%s request successfully sent to %s. Response: %s %s" % (method, url, res.status_code, res.reason)) + with open('/app/reload_successful.txt', 'w', encoding='utf-8') as f: + f.write("true") + else: + logger.error("%s request failed to send to %s. Response: %s %s" % (method, url, res.status_code, res.reason)) + with open('/app/reload_successful.txt', 'w', encoding='utf-8') as f: + f.write("false") return res diff --git a/sidecar/resources.py b/sidecar/resources.py index ca67f286..2b204d73 100644 --- a/sidecar/resources.py +++ b/sidecar/resources.py @@ -20,7 +20,7 @@ } -def _get_file_data_and_name(full_filename, content, resource): +def _get_file_data_and_name(full_filename, content, resource, logger): if resource == "secret": file_data = base64.b64decode(content).decode() else: @@ -28,14 +28,14 @@ def _get_file_data_and_name(full_filename, content, resource): if full_filename.endswith(".url"): filename = full_filename[:-4] - file_data = request(file_data, "GET").text + file_data = request(file_data, "GET", logger,).text else: filename = full_filename return filename, file_data -def listResources(label, targetFolder, url, method, payload, current, folderAnnotation, resource): +def listResources(label, targetFolder, url, method, payload, current, folderAnnotation, resource, logger): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current) if namespace == "ALL": @@ -48,30 +48,30 @@ def listResources(label, targetFolder, url, method, payload, current, folderAnno metadata = sec.metadata if metadata.labels is None: continue - print(f'Working on {resource}: {metadata.namespace}/{metadata.name}') + logger.info("Working on %s: %s/%s" % (resource, metadata.namespace, metadata.name)) if label in sec.metadata.labels.keys(): - print(f"Found {resource} with label") + logger.info("Found %s with label" % resource) if sec.metadata.annotations is not None: if folderAnnotation in sec.metadata.annotations.keys(): destFolder = sec.metadata.annotations[folderAnnotation] dataMap = sec.data if dataMap is None: - print(f"No data field in {resource}") + logger.info("No data field in %s" % resource) continue if label in sec.metadata.labels.keys(): for data_key in dataMap.keys(): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], - resource) + resource, logger) writeTextToFile(destFolder, filename, filedata) if url is not None: - request(url, method, payload) + request(url, method, logger, payload) def _watch_resource_iterator(label, targetFolder, url, method, payload, - current, folderAnnotation, resource): + current, folderAnnotation, resource, logger): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", current) if namespace == "ALL": @@ -84,82 +84,81 @@ def _watch_resource_iterator(label, targetFolder, url, method, payload, metadata = event['object'].metadata if metadata.labels is None: continue - print(f'Working on {resource} {metadata.namespace}/{metadata.name}') + logger.info("Working on %s %s/%s" % (resource, metadata.namespace, metadata.name)) if label in event['object'].metadata.labels.keys(): - print(f"{resource} with label found") + logger.info("%s with label found" % resource) if event['object'].metadata.annotations is not None: if folderAnnotation in event['object'].metadata.annotations.keys(): destFolder = event['object'].metadata.annotations[folderAnnotation] - print('Found a folder override annotation, ' - f'placing the {resource} in: {destFolder}') + logger.info("Found a folder override annotation, placing the %s in: %s" % (resource, destFolder)) dataMap = event['object'].data if dataMap is None: - print(f"{resource} does not have data.") + logger.info("%s does not have data." % resource) continue eventType = event['type'] for data_key in dataMap.keys(): - print(f"File in {resource} {data_key} {eventType}") - + logger.info("File in %s %s %s " % (resource, data_key, eventType)) if (eventType == "ADDED") or (eventType == "MODIFIED"): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], - resource) + resource, logger) writeTextToFile(destFolder, filename, filedata) if url is not None: - request(url, method, payload) + request(url, method, logger, payload) else: filename = data_key[:-4] if data_key.endswith(".url") else data_key - removeFile(destFolder, filename) + removeFile(destFolder, filename, logger) if url is not None: - request(url, method, payload) + request(url, method, logger, payload) -def _watch_resource_loop(*args): +def _watch_resource_loop(logger, *args): while True: try: _watch_resource_iterator(*args) except ApiException as e: if e.status != 500: - print(f"ApiException when calling kubernetes: {e}\n") + logger.error("ApiException when calling kubernetes: %s" % e) else: raise except ProtocolError as e: - print(f"ProtocolError when calling kubernetes: {e}\n") + logger.error("ProtocolError when calling kubernetes: %s" % e) except Exception as e: - print(f"Received unknown exception: {e}\n") + logger.error("Received unknown exception: %s" % e) def watchForChanges(label, targetFolder, url, method, payload, - current, folderAnnotation, resources): + current, folderAnnotation, resources, logger): firstProc = Process(target=_watch_resource_loop, - args=(label, targetFolder, url, method, payload, - current, folderAnnotation, resources[0]) + args=(logger, label, targetFolder, url, method, payload, + current, folderAnnotation, resources[0], logger) ) firstProc.start() if len(resources) == 2: secProc = Process(target=_watch_resource_loop, - args=(label, targetFolder, url, method, payload, - current, folderAnnotation, resources[1]) + args=(logger, label, targetFolder, url, method, payload, + current, folderAnnotation, resources[1], logger) ) secProc.start() while True: if not firstProc.is_alive(): - print(f"Process for {resources[0]} died. Stopping and exiting") + logger.info("Process for %s died. Stopping and exiting" % resources[0]) if len(resources) == 2 and secProc.is_alive(): secProc.terminate() elif len(resources) == 2: - print(f"Process for {resources[1]} also died...") + logger.info("Process for %s also died..." % resources[1]) raise Exception("Loop died") if len(resources) == 2 and not secProc.is_alive(): - print(f"Process for {resources[1]} died. Stopping and exiting") + logger.info("Process for %s died. Stopping and exiting" % resources[1]) if firstProc.is_alive(): firstProc.terminate() else: - print(f"Process for {resources[0]} also died...") + pass + logger.info("Process for %s also died..." % resources[0]) raise Exception("Loop died") sleep(5) diff --git a/sidecar/sidecar.py b/sidecar/sidecar.py index 03002d5c..c32c4ae7 100644 --- a/sidecar/sidecar.py +++ b/sidecar/sidecar.py @@ -1,38 +1,51 @@ -import os - +import os, logging, sys from kubernetes import client, config from resources import listResources, watchForChanges +def setup_custom_logger(name): + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + handler = logging.FileHandler('log.txt', mode='w') + handler.setFormatter(formatter) + screen_handler = logging.StreamHandler(stream=sys.stdout) + screen_handler.setFormatter(formatter) + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + logger.addHandler(handler) + logger.addHandler(screen_handler) + return logger def main(): - print("Starting collector") - + logger = setup_custom_logger('sidecar') + if os.path.exists('/app/reload_successful.txt'): + os.remove('/app/reload_successful.txt') + with open('/app/reload_successful.txt', 'w', encoding='utf-8') as f: + f.write("true") + logger.info("Starting collector") folderAnnotation = os.getenv('FOLDER_ANNOTATIONS') if folderAnnotation is None: - print("No folder annotation was provided, defaulting to k8s-sidecar-target-directory") + logger.info("No folder annotation was provided, defaulting to k8s-sidecar-target-directory") folderAnnotation = "k8s-sidecar-target-directory" label = os.getenv('LABEL') if label is None: - print("Should have added LABEL as environment variable! Exit") + logger.error("Should have added LABEL as environment variable! Exit") return -1 targetFolder = os.getenv('FOLDER') if targetFolder is None: - print("Should have added FOLDER as environment variable! Exit") + logger.error("Should have added FOLDER as environment variable! Exit") return -1 resources = os.getenv('RESOURCE', 'configmap') resources = ("secret", "configmap") if resources == "both" else (resources, ) - print(f"Selected resource type: {resources}") - + logger.info("Selected resource type: %s" % resources) method = os.getenv('REQ_METHOD') url = os.getenv('REQ_URL') payload = os.getenv('REQ_PAYLOAD') - config.load_incluster_config() - print("Config for cluster api loaded...") + logger.info("Config for cluster api loaded...") namespace = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace").read() if os.getenv('SKIP_TLS_VERIFY') == 'true': @@ -44,11 +57,10 @@ def main(): if os.getenv("METHOD") == "LIST": for res in resources: listResources(label, targetFolder, url, method, payload, - namespace, folderAnnotation, res) + namespace, folderAnnotation, res, logger) else: watchForChanges(label, targetFolder, url, method, - payload, namespace, folderAnnotation, resources) - + payload, namespace, folderAnnotation, resources, logger) if __name__ == '__main__': main() \ No newline at end of file