From c4a1b7eed4306e9bd8fc2cfe96577e86f3405a2d Mon Sep 17 00:00:00 2001 From: Marius Oehler Date: Mon, 7 Sep 2020 16:49:13 +0200 Subject: [PATCH] Changed logging format --- .gitignore | 4 ++- .idea/.gitignore | 3 --- config.yml | 17 ------------- run.py | 46 ++++++++++++++++++++++++++++++++-- script/core.py | 22 +++++++++++++---- script/grafana.py | 61 ++++++++++++++++++---------------------------- script/ldap.py | 3 +-- setup.py | 1 - tests/test_core.py | 8 +++--- 9 files changed, 93 insertions(+), 72 deletions(-) delete mode 100644 .idea/.gitignore diff --git a/.gitignore b/.gitignore index bba59db..74b4bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,6 @@ dmypy.json # Project-specific .lock -.idea \ No newline at end of file +.idea +/shelf/ +/workspace.xml \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 26d3352..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml diff --git a/config.yml b/config.yml index 02edf7e..89413ab 100644 --- a/config.yml +++ b/config.yml @@ -1,56 +1,39 @@ config: - grafana: # URL of the target grafana-server. url: localhost:3000 - # User account name with admin rights on target grafana-server. user: admin - # Password of account with admin rights on target grafana-server. password: admin ldap: - # Set to True if NTLM should be used for LDAP. useNTLM: True - # If True, SSL will be used for connection to LDAP. useSSL: False - # URL of the source LDAP-Server. url: ldap.forumsys.com - # Port of the ldap instance provided in the url-variable. port: 389 - # Group search-base of the source LDAP-Server. groupSearchBase: dc=example,dc=com - # Filter that should be used for the group search. groupSearchFilter: - # Search-Base for user objects on the LDAP-Server. userSearchBase: dc=example,dc=com - # Filter that should be used for user searches. userSearchFilter: - # Attribute name under which the members of a group object can be found. memberAttributeName: uniqueMember - # Name of the attribute which should be used as login in grafana. userLoginAttribute: uid - # Name of the attribute which should be used as Name in grafana. userNameAttribute: cn - # Name of the attribute which should be used as mail in grafana. userMailAttribute: mail - # Login for the LDAP-Server. login: test\cn=read-only-admin,dc=example,dc=com - # Password for the LDAP-Server. password: password diff --git a/run.py b/run.py index 979db49..03f817f 100644 --- a/run.py +++ b/run.py @@ -1,5 +1,42 @@ -from script.core import export +from script.core import startUserSync import argparse +import logging + +class DispatchingFormatter: + def __init__(self, formatters, default_formatter): + self._formatters = formatters + self._default_formatter = default_formatter + + def format(self, record): + formatter = self._formatters.get(record.name, self._default_formatter) + return formatter.format(record) + +def setup_logger(): + """ + Setting up the used logger. The 'mutate' logger will print whether dry-run is used and changes are being applied. + """ + log_format = '%(asctime)s - %(levelname)s - %(module)7s - %(message)s' + log_format_mut = log_format + + if args.dry_run: + log_format_mut = '%(asctime)s - %(levelname)s - %(module)7s - [SKIPPED] %(message)s' + else: + log_format_mut = log_format + + + logger = logging.getLogger() + while logger.handlers: + logger.handlers.pop() + + formatter = DispatchingFormatter({ + 'mutate': logging.Formatter(log_format_mut), + }, + logging.Formatter(log_format) + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.setLevel(logging.INFO) + logger.addHandler(handler) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Process config path') @@ -10,4 +47,9 @@ 'changes are printed to the console.', action='store_true') args = parser.parse_args() - export(args.config_path, args.bind, args.dry_run) + + # setup the logger + setup_logger() + + # starts the sync process + startUserSync(args.config_path, args.bind, args.dry_run) diff --git a/script/core.py b/script/core.py index 9afe1a8..90aa7c2 100644 --- a/script/core.py +++ b/script/core.py @@ -8,8 +8,7 @@ from requests.exceptions import ConnectionError from .config import * -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("grafana-ldap-sync-script") +logger = logging.getLogger() PERMISSION_MAP = { "View": 1, @@ -237,7 +236,7 @@ def remove_unused_items(team_mappings): delete_unmapped_users(team_mappings) -def export(config_path, bind, dry_run): +def startUserSync(config_path, bind, dry_run): """ Checks if a .lock file is currently present. If no .lock file is present, the updating of the grafana teams, folders and users is performed. @@ -245,18 +244,31 @@ def export(config_path, bind, dry_run): """ global configuration if lock(): - logger.info("Starting task...") try: + logger.info("=================================================") + logger.info("Starting user synchronization...") + configuration = config(config_path) + configuration.DRY_RUN = dry_run if configuration.DRY_RUN: - print("dryRun enabled: Changes will not be applied!") + logger.info("!! DryRun enabled: Changes will not be applied !!") + + logger.info("=================================================") + + logger.info("Setting up the connection to the Grafana server..") setup_grafana(configuration) + logger.info("Setting up the connection to the LDAP server..") setup_ldap(configuration) + logger.info("Reading the user and team mappings..") mapping = read_mapping_from_csv(bind) + logger.info("Updating the Grafana teams..") update_teams(mapping["teams"]) + logger.info("Updating the Grafana folders..") update_folders(mapping["folders"]) + logger.info("Removing unused teams and users..") remove_unused_items(mapping["teams"]) + logger.info("Task finished successfully!") except LDAPSocketOpenError: logger.error("Task aborted, unable to reach LDAP-Server.") diff --git a/script/grafana.py b/script/grafana.py index d9ea9a5..d06ed51 100644 --- a/script/grafana.py +++ b/script/grafana.py @@ -2,12 +2,13 @@ from grafana_api.grafana_face import GrafanaFace from .config import * from .helpers import * +import logging grafana_api = "" configuration = "" -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("grafana-ldap-sync-script") +logger = logging.getLogger() +logger_mut = logging.getLogger("mutate") def setup_grafana(config_dict): global grafana_api, configuration @@ -27,10 +28,8 @@ def delete_team_by_name(name): team_data = grafana_api.teams.get_team_by_name(name) if len(team_data) > 0: for data_set in team_data: - if configuration.DRY_RUN: - logger.info("Would have deleted team with name: %s and id: %s" % (name, data_set["id"])) - else: - logger.info("Deleting team with name %s and id %s" % (name, data_set["id"])) + logger_mut.info("Deleting team with name %s and id %s" % (name, data_set["id"])) + if not configuration.DRY_RUN: grafana_api.teams.delete_team(data_set["id"]) return True return False @@ -43,10 +42,8 @@ def create_team(name, mail): :param mail: The mail of the team. :return: The API response. """ - if configuration.DRY_RUN: - logger.info("Would have created team with name: %s" % name) - else: - logger.info("Creating team with name %s" % name) + logger_mut.info("Creating team with name %s" % name) + if not configuration.DRY_RUN: return grafana_api.teams.add_team({ "name": name, "mail": mail @@ -61,12 +58,10 @@ def create_user_with_random_pw(user): user_dict = dict(user) user_dict["password"] = get_random_alphanumerical() user_dict["OrgId"] = 1 - if configuration.DRY_RUN: - logger.info("Would have created user with json %s" % str(user_dict)) - else: - logger.info("Creating user with login %s, name %s and mail %s" % - (user_dict["login"], user_dict["name"], user_dict["email"]) - ) + + logger_mut.info("Creating user with login %s, name %s and mail %s" % (user_dict["login"], user_dict["name"], user_dict["email"])) + + if not configuration.DRY_RUN: grafana_api.admin.create_user(user_dict) @@ -76,11 +71,11 @@ def delete_user_by_login(login): :param login: The login of the user to be deleted. :return: The response of the api. """ - if not login == "admin": - if configuration.DRY_RUN: - logger.info("Would have deleted user with name: %s" % login) - else: - logger.info("Deleting user with name %s" % login) + if login == configuration.GRAFANA_AUTH[0]: + logger.info("The user '%s' is used by this script for accessing Grafana thus will not be deleted." % login) + else: + logger_mut.info("Deleting user with name %s" % login) + if not configuration.DRY_RUN: return grafana_api.admin.delete_user(grafana_api.users.find_user(login)["id"]) return False @@ -94,10 +89,8 @@ def create_folder(folder_name, folder_uuid): :return: The api-response if the folder was create successfully. If an error occurs, false is returned. """ try: - if configuration.DRY_RUN: - logger.info("Would have created folder with name: %s and id: %s" % (folder_name, folder_uuid)) - else: - logger.info("Creating folder with name %s and id %s" % (folder_name, folder_uuid)) + logger_mut.info("Creating folder with name %s and id %s" % (folder_name, folder_uuid)) + if not configuration.DRY_RUN: return grafana_api.folder.create_folder(folder_name, folder_uuid) except GrafanaClientError: return False @@ -110,10 +103,8 @@ def add_user_to_team(login, team): :param team: The team the user should be added to. """ try: - if configuration.DRY_RUN: - logger.info("Would have added user %s to team %s" % (login, team)) - else: - logger.info("Adding user %s to team %s" % (login, team)) + logger_mut.info("Adding user %s to team %s" % (login, team)) + if not configuration.DRY_RUN: grafana_api.teams.add_team_member(get_id_of_team(team), get_id_by_login(login)) except GrafanaBadInputError: return False @@ -142,10 +133,8 @@ def get_members_of_team(team): def remove_member_from_team(grafana_team, user_login): - if configuration.DRY_RUN: - print("Would have removed user %s from team %s" % (user_login, grafana_team)) - else: - logger.info("Removing user %s from team %s" % (user_login, grafana_team)) + logger_mut.info("Removing user %s from team %s" % (user_login, grafana_team)) + if not configuration.DRY_RUN: grafana_api.teams.remove_team_member(get_id_of_team(grafana_team), get_id_by_login(user_login)) @@ -200,10 +189,8 @@ def update_folder_permissions(folder_id, permissions): """ Sets the given permissions for the folder found under the given id """ - if configuration.DRY_RUN: - logger.info("Would have set permission of folder %s to %s" % (folder_id, permissions)) - else: - logger.info("Setting permission of folder %s to %s" % (folder_id, permissions)) + logger_mut.info("Setting permission of folder %s to %s" % (folder_id, permissions)) + if not configuration.DRY_RUN: grafana_api.folder.update_folder_permissions(folder_id, {"items": permissions}) diff --git a/script/ldap.py b/script/ldap.py index e1e81fc..29b56a2 100644 --- a/script/ldap.py +++ b/script/ldap.py @@ -4,8 +4,7 @@ from .config import config from .helpers import * -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("grafana-ldap-sync-script") +logger = logging.getLogger() configuration = "" user_cache = {} diff --git a/setup.py b/setup.py index 07fd8d5..3180476 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ from setuptools import setup, find_packages - setup( name='grafana-ldap-sync-script', version='0.1.0', diff --git a/tests/test_core.py b/tests/test_core.py index e0908d0..8a1ba89 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -379,7 +379,7 @@ def test_locks_and_unlocks(self, mock_setup_ldap, mock_unlock, mock_remove_unuse mock_config.return_value = True mock_lock.return_value = True - core.export("") + core.startUserSync("") self.assertEqual(mock_lock.call_count, 1) self.assertEqual(mock_unlock.call_count, 1) @@ -405,7 +405,7 @@ def test_locks_and_unlocks_on_connection_error(self, mock_setup_ldap, mock_unloc mock_config.return_value = True mock_lock.return_value = True - core.export("") + core.startUserSync("") self.assertEqual(mock_lock.call_count, 1) self.assertEqual(mock_unlock.call_count, 1) @@ -432,7 +432,7 @@ def test_locks_and_unlocks_on_LDAPSocketOpenError(self, mock_setup_ldap, mock_un mock_config.return_value = True mock_lock.return_value = True - core.export("") + core.startUserSync("") self.assertEqual(mock_lock.call_count, 1) self.assertEqual(mock_unlock.call_count, 1) @@ -457,7 +457,7 @@ def test_nothing_called_when_locked(self, mock_unlock, mock_remove_unused_items, mock_config.return_value = True mock_lock.return_value = False - core.export("") + core.startUserSync("") self.assertEqual(mock_lock.call_count, 1) self.assertFalse(mock_remove_unused_items.called)