From e612815f5c3ac83e401e759bd88502d3ea7aa609 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 13:48:43 +0200 Subject: [PATCH 01/32] have generic python module name --- bin/ams-alarm-to-queue | 2 +- bin/ams-metric-to-queue | 2 +- bin/ams-publisherd | 10 +++++----- pymod/alarmtoqueue.py | 4 ++-- pymod/consume.py | 6 +++--- pymod/metrictoqueue.py | 4 ++-- pymod/publish.py | 4 ++-- pymod/purge.py | 2 +- pymod/run.py | 10 +++++----- pymod/stats.py | 2 +- setup.py | 4 ++-- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/bin/ams-alarm-to-queue b/bin/ams-alarm-to-queue index c4e6687..9c2a1fb 100755 --- a/bin/ams-alarm-to-queue +++ b/bin/ams-alarm-to-queue @@ -1,5 +1,5 @@ #!/usr/bin/python3 -from argo_nagios_ams_publisher import alarmtoqueue +from ams_publisher import alarmtoqueue alarmtoqueue.main() diff --git a/bin/ams-metric-to-queue b/bin/ams-metric-to-queue index 5a9c4b0..01811cb 100755 --- a/bin/ams-metric-to-queue +++ b/bin/ams-metric-to-queue @@ -1,5 +1,5 @@ #!/usr/bin/python3 -from argo_nagios_ams_publisher import metrictoqueue +from ams_publisher import metrictoqueue metrictoqueue.main() diff --git a/bin/ams-publisherd b/bin/ams-publisherd index 5dfe322..ec55f57 100755 --- a/bin/ams-publisherd +++ b/bin/ams-publisherd @@ -5,11 +5,11 @@ import pwd import signal import sys -from argo_nagios_ams_publisher.config import parse_config -from argo_nagios_ams_publisher.log import Logger -from argo_nagios_ams_publisher.run import init_dirq_consume -from argo_nagios_ams_publisher.stats import query_stats, setup_statssocket -from argo_nagios_ams_publisher.shared import Shared +from ams_publisher.config import parse_config +from ams_publisher.log import Logger +from ams_publisher.run import init_dirq_consume +from ams_publisher.stats import query_stats, setup_statssocket +from ams_publisher.shared import Shared def get_userids(user): diff --git a/pymod/alarmtoqueue.py b/pymod/alarmtoqueue.py index 27b6b3c..257a830 100644 --- a/pymod/alarmtoqueue.py +++ b/pymod/alarmtoqueue.py @@ -4,8 +4,8 @@ from messaging.error import MessageError from messaging.queue.dqs import DQS -from argo_nagios_ams_publisher import config -from argo_nagios_ams_publisher import log +from ams_publisher import config +from ams_publisher import log import argparse import datetime diff --git a/pymod/consume.py b/pymod/consume.py index a880e35..6ebab11 100644 --- a/pymod/consume.py +++ b/pymod/consume.py @@ -6,9 +6,9 @@ from messaging.queue.dqs import DQS from multiprocessing import Process -from argo_nagios_ams_publisher.purge import Purger -from argo_nagios_ams_publisher.shared import Shared -from argo_nagios_ams_publisher.stats import StatSig +from ams_publisher.purge import Purger +from ams_publisher.shared import Shared +from ams_publisher.stats import StatSig class ConsumerQueue(StatSig, Process): diff --git a/pymod/metrictoqueue.py b/pymod/metrictoqueue.py index 950eb2e..5ec90d7 100644 --- a/pymod/metrictoqueue.py +++ b/pymod/metrictoqueue.py @@ -4,8 +4,8 @@ from messaging.error import MessageError from messaging.queue.dqs import DQS -from argo_nagios_ams_publisher import config -from argo_nagios_ams_publisher import log +from ams_publisher import config +from ams_publisher import log import argparse import os diff --git a/pymod/publish.py b/pymod/publish.py index 1199a03..d6d5be7 100644 --- a/pymod/publish.py +++ b/pymod/publish.py @@ -7,8 +7,8 @@ from argo_ams_library.ams import ArgoMessagingService from argo_ams_library.amsmsg import AmsMessage -from argo_nagios_ams_publisher.shared import Shared -from argo_nagios_ams_publisher.stats import StatSig +from ams_publisher.shared import Shared +from ams_publisher.stats import StatSig from argo_ams_library.amsexceptions import AmsConnectionException, AmsServiceException diff --git a/pymod/purge.py b/pymod/purge.py index d9327d6..e596fd7 100644 --- a/pymod/purge.py +++ b/pymod/purge.py @@ -1,6 +1,6 @@ import threading import time -from argo_nagios_ams_publisher.shared import Shared +from ams_publisher.shared import Shared from datetime import datetime diff --git a/pymod/run.py b/pymod/run.py index d1daa88..aad2ea7 100644 --- a/pymod/run.py +++ b/pymod/run.py @@ -2,11 +2,11 @@ import datetime import time -from argo_nagios_ams_publisher.publish import FilePublisher, MessagingPublisher -from argo_nagios_ams_publisher.consume import ConsumerQueue -from argo_nagios_ams_publisher.stats import StatSock -from argo_nagios_ams_publisher.shared import Shared -from argo_nagios_ams_publisher.config import parse_config +from ams_publisher.publish import FilePublisher, MessagingPublisher +from ams_publisher.consume import ConsumerQueue +from ams_publisher.stats import StatSock +from ams_publisher.shared import Shared +from ams_publisher.config import parse_config from multiprocessing import Event, Lock, Value, Manager from threading import Event as ThreadEvent diff --git a/pymod/stats.py b/pymod/stats.py index 633cec8..ee13d33 100644 --- a/pymod/stats.py +++ b/pymod/stats.py @@ -9,7 +9,7 @@ from threading import Thread from multiprocessing import Process -from argo_nagios_ams_publisher.shared import Shared +from ams_publisher.shared import Shared MAXCMDLENGTH = 128 STATSOCK = '/run/argo-nagios-ams-publisher/sock' diff --git a/setup.py b/setup.py index b3988b3..4eaa4be 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ def get_ver(): description='Bridge from Nagios to the ARGO Messaging system', long_description='Bridge from Nagios to the ARGO Messaging system', url='https://github.com/ARGOeu/argo-nagios-ams-publisher', - package_dir={'argo_nagios_ams_publisher': 'pymod/'}, - packages=['argo_nagios_ams_publisher'], + package_dir={'ams_publisher': 'pymod/'}, + packages=['ams_publisher'], data_files=[('/etc/argo-nagios-ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), ('/usr/lib/systemd/system/', ['init/ams-publisher.service']) if is_c7() else \ ('/etc/init.d/', ['init/ams-publisher'])], From 12f7297b7120d87084c18ec72d91032402e05cc0 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 13:52:03 +0200 Subject: [PATCH 02/32] generic spec name --- argo-nagios-ams-publisher.spec => ams-publisher.spec | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename argo-nagios-ams-publisher.spec => ams-publisher.spec (100%) diff --git a/argo-nagios-ams-publisher.spec b/ams-publisher.spec similarity index 100% rename from argo-nagios-ams-publisher.spec rename to ams-publisher.spec From 9763ef55b6624414b59997411058f2a981777161 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 14:04:00 +0200 Subject: [PATCH 03/32] do not create user for runtime permissions in spec --- ams-publisher.spec | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index c31f5de..60524a3 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -64,16 +64,6 @@ rm -rf $RPM_BUILD_ROOT %preun %systemd_preun ams-publisher.service -%pre -if ! /usr/bin/id nagios &>/dev/null; then - /usr/sbin/useradd -r -m -d /var/log/nagios -s /bin/sh -c "nagios" nagios || \ - logger -t nagios/rpm "Unexpected error adding user \"nagios\". Aborting installation." -fi -if ! /usr/bin/getent group nagiocmd &>/dev/null; then - /usr/sbin/groupadd nagiocmd &>/dev/null || \ - logger -t nagios/rpm "Unexpected error adding group \"nagiocmd\". Aborting installation." -fi - %changelog * Mon Feb 1 2021 Daniel Vrcic - 0.3.9-1%{?dist} - ARGO-2855 ams-publisher py3 switch From ad5828f376e66959d031fdbb4baa39edeb9d1156 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 16:01:07 +0200 Subject: [PATCH 04/32] remove hardcoded argo-nagios-ams-publisher and have generic ams-publisher paths --- README.md | 32 ++++++++++++++++---------------- config/ams-publisher.conf | 2 +- pymod/alarmtoqueue.py | 2 +- pymod/config.py | 2 +- pymod/metrictoqueue.py | 2 +- setup.py | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b8ecc2a..2f202a1 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # argo-nagios-ams-publisher -## Description +## Description `argo-nagios-ams-publisher` is a component acting as bridge from Nagios to ARGO Messaging system. It's essential part of software stack running on ARGO monitoring instance and is responsible for forming and dispatching messages that wrap up results of Nagios probes/tests. It is running as a unix daemon and it consists of two subsystems: -- queueing mechanism +- queueing mechanism - publishing/dispatching part -Messages are cached in local directory queue with the help of OCSP Nagios commands and each queue is being monitored and consumed by the daemon. After configurable amount of accumulated messages, publisher that is associated to queue sends them to ARGO Messaging system and drains the queue. `argo-nagios-ams-publisher` is written in multiprocessing manner so there is support for multiple (consume, publish) pairs where for each, new worker process will be spawned. +Messages are cached in local directory queue with the help of OCSP Nagios commands and each queue is being monitored and consumed by the daemon. After configurable amount of accumulated messages, publisher that is associated to queue sends them to ARGO Messaging system and drains the queue. `argo-nagios-ams-publisher` is written in multiprocessing manner so there is support for multiple (consume, publish) pairs where for each, new worker process will be spawned. Filling and draining of directory queue is asynchronous. Nagios delivers results on its own constant rate while `argo-nagios-ams-publisher` consume and publish them on its own configurable constant rate. It's important to keep the two rates close enough so that the results don't pile up in the queue and leave it early. Component has a mechanism of inspection of rates and trends over time to keep the constants in sync. Also it's resilient to network issues so it will retry configurable number of times to send a messages to ARGO Messaging system. It's also important to note that consume and publish of the queue is a serial process so if publish is stopped, consume part of the worker will be also stopped. That could lead to pile up of results in the queue and since every result is represented as a one file on the file system, easily exhaustion of free inodes and therefore unusable monitoring instance. @@ -23,7 +23,7 @@ Complete list of features are: - configurable bulk of messages sent to ARGO Messaging system - configurable retry attempts in case of network connection problems - purger that will keep queue only with sound data -- message rate inspection of each worker for monitoring purposes +- message rate inspection of each worker for monitoring purposes ## Installation @@ -32,17 +32,17 @@ Component is supported on CentOS 6 and CentOS 7. RPM packages and all needed dep yum install -y argo-nagios-ams-publisher Component relies on: -- `argo-ams-library` - interaction with ARGO Messaging +- `argo-ams-library` - interaction with ARGO Messaging - `avro` - avro serialization of messages' payload - `python-argparse` - ease build and parse of command line arguments -- `python-daemon` - ease daemonizing of component -- `python-messaging` - CERN's library for directory based caching/queueing +- `python-daemon` - ease daemonizing of component +- `python-messaging` - CERN's library for directory based caching/queueing - `pytz` - timezone manipulation | File Types | Destination | |-------------------|----------------------------------------------------| -| Configuration | `/etc/argo-nagios-ams-publisher/ams-publisher.conf`| +| Configuration | `/etc/ams-publisher/ams-publisher.conf`| | Daemon component | `/usr/bin/ams-publisherd` | | Cache delivery | `/usr/bin/ams-alarm-to-queue, ams-metric-to-queue` | | Init script (C6) | `/etc/init.d/ams-publisher` | @@ -53,12 +53,12 @@ Component relies on: ## Configuration -Central configuration is in `ams-publisher.conf`. Configuration consists of `[General]` section and `[Queue_], [Topic_]` section pairs. +Central configuration is in `ams-publisher.conf`. Configuration consists of `[General]` section and `[Queue_], [Topic_]` section pairs. ### General section ``` -[General] +[General] Host = NAGIOS.FQDN.EXAMPLE.COM RunAsUser = nagios StatsEveryHour = 24 @@ -71,8 +71,8 @@ StatSocket = /var/run/argo-nagios-ams-publisher/sock * `Host` - FQDN of ARGO Monitoring instance that will be part of formed messages dispatched to ARGO Messaging system * `RunAsUser` - component will run with effective UID and GID of given user, usually `nagios` -* `StatsEveryHour` - write periodic report in system logs. Example is given in [Running](Running) -* `PublishMsgFile`, `PublishMsgFileDir` - "file publisher" that is actually only for testing purposes. If enabled, messages will not be dispatched to ARGO Messaging System, instead it will just be appended to plain text file +* `StatsEveryHour` - write periodic report in system logs. Example is given in [Running](Running) +* `PublishMsgFile`, `PublishMsgFileDir` - "file publisher" that is actually only for testing purposes. If enabled, messages will not be dispatched to ARGO Messaging System, instead it will just be appended to plain text file * `TimeZone` - construct timestamp of messages within specified timezone * `StatsSocket` - query socket that is used for inspection of rates of each worker. It used by the `ams-publisher` Nagios probe. @@ -99,15 +99,15 @@ Topic = metric_data Bulksize = 100 MsgType = metric_data Avro = True -AvroSchema = /etc/argo-nagios-ams-publisher/metric_data.avsc +AvroSchema = /etc/ams-publisher/metric_data.avsc Retry = 5 Timeout = 60 -SleepRetry = 300 +SleepRetry = 300 ``` * `[Queue_Metrics].Directory` - path of directory queue on the filesystem where local cache delivery tools write results of Nagios tests/probes. * `[Queue_Metrics].Rate` - local cache inspection rate. 10 means that cache will be inspected 10 times at a second because thats the number of status results expected from Nagios that will be picked up verly early. For low volume tenants this could be a lower number. -* `[Queue_Metrics].Purge,PurgeEverySec,MaxTemp,MaxLock` - purge the staled elements of directory queue every `PurgeEverySec` seconds. It cleans the empty intermediate directories below directory queue path, temporary results that exceeded `MaxTemp` time and locked results that exceeded `MaxLock`. +* `[Queue_Metrics].Purge,PurgeEverySec,MaxTemp,MaxLock` - purge the staled elements of directory queue every `PurgeEverySec` seconds. It cleans the empty intermediate directories below directory queue path, temporary results that exceeded `MaxTemp` time and locked results that exceeded `MaxLock`. > It is advisable to leave `MaxLock = 0` which skips every result that have been transformed into a message and added into in-memory queue, but had not yet been dispatched. * `[Queue_Metrics].Granularity` - new intermediate directory in the toplevel directory queue path is created every `Granularity` seconds * `[Topic_Metrics].Host,Key,Project,Topic` - options needed for delivering of messages to ARGO Messaging system. `Host` designates the FQDN, `Key` is authorization token, `Project` represents a tenant name and `Topic` is final destination scoped to tenant @@ -130,7 +130,7 @@ CentOS 7: systemctl start ams-publisher.service ``` -Component periodically reports rates of each worker in system logs. It does so every `StatsEveryHour`. +Component periodically reports rates of each worker in system logs. It does so every `StatsEveryHour`. ``` 2020-04-08 08:53:42 ams-publisher[963]: INFO - Periodic report (every 6.0h) 2020-04-08 08:53:42 ams-publisher[983]: INFO - ConsumerQueue metrics: consumed 45787 msgs in 6.00 hours diff --git a/config/ams-publisher.conf b/config/ams-publisher.conf index 777666d..a0694b2 100644 --- a/config/ams-publisher.conf +++ b/config/ams-publisher.conf @@ -24,7 +24,7 @@ Bulksize = 100 Topic = metric_data MsgType = metric_data Avro = True -AvroSchema = /etc/argo-nagios-ams-publisher/metric_data.avsc +AvroSchema = /etc/ams-publisher/metric_data.avsc Retry = 5 Timeout = 60 SleepRetry = 300 diff --git a/pymod/alarmtoqueue.py b/pymod/alarmtoqueue.py index 257a830..82cdd9e 100644 --- a/pymod/alarmtoqueue.py +++ b/pymod/alarmtoqueue.py @@ -15,7 +15,7 @@ import sys -CONF = '/etc/argo-nagios-ams-publisher/ams-publisher.conf' +CONF = '/etc/ams-publisher/ams-publisher.conf' def seteuser(user): diff --git a/pymod/config.py b/pymod/config.py index 6e957ea..26cc01a 100644 --- a/pymod/config.py +++ b/pymod/config.py @@ -2,7 +2,7 @@ import sys from pytz import timezone, UnknownTimeZoneError -conf = '/etc/argo-nagios-ams-publisher/ams-publisher.conf' +conf = '/etc/ams-publisher/ams-publisher.conf' def get_queue_granul(queue): diff --git a/pymod/metrictoqueue.py b/pymod/metrictoqueue.py index 5ec90d7..99d81f1 100644 --- a/pymod/metrictoqueue.py +++ b/pymod/metrictoqueue.py @@ -14,7 +14,7 @@ import datetime import pytz -CONF = '/etc/argo-nagios-ams-publisher/ams-publisher.conf' +CONF = '/etc/ams-publisher/ams-publisher.conf' def seteuser(user): diff --git a/setup.py b/setup.py index 4eaa4be..3914ec6 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def get_ver(): url='https://github.com/ARGOeu/argo-nagios-ams-publisher', package_dir={'ams_publisher': 'pymod/'}, packages=['ams_publisher'], - data_files=[('/etc/argo-nagios-ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), + data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), ('/usr/lib/systemd/system/', ['init/ams-publisher.service']) if is_c7() else \ ('/etc/init.d/', ['init/ams-publisher'])], scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', From 140d8f090e290abd80b7b10843fc50e2603319ab Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 16:04:26 +0200 Subject: [PATCH 05/32] remove Centos 6 from doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f202a1..b114c0a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Complete list of features are: ## Installation -Component is supported on CentOS 6 and CentOS 7. RPM packages and all needed dependencies are available in ARGO repositories so installation of component simply narrows down to installing a package: +Component is supported on CentOS 7. RPM packages and all needed dependencies are available in ARGO repositories so installation of component simply narrows down to installing a package: yum install -y argo-nagios-ams-publisher From 4836a415b48523002d597e04e0ccc38d4b4cd93c Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 18:21:27 +0200 Subject: [PATCH 06/32] refine spec with two package definitions --- ams-publisher.spec | 50 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index 60524a3..9dba64e 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -4,10 +4,10 @@ %define stripc() %(echo %1 | sed 's/el7.centos/el7/') %define mydist %{stripc %{dist}} -Name: argo-nagios-ams-publisher +Name: ams-publisher +Summary: Bridge from Nagios/Sensu to the ARGO Messaging system Version: 0.3.9 Release: 1%{mydist} -Summary: Bridge from Nagios to the ARGO Messaging system Group: Network/Monitoring License: ASL 2.0 @@ -27,10 +27,44 @@ Requires(post): systemd Requires(preun): systemd Requires(postun): systemd - %description +Bridge from Nagios/Sensu to the ARGO Messaging system + + +%package -n argo-sensu-%{name} +Summary: Bridge from Sensu to the ARGO Messaging system + +%description -n argo-sensu-%{name} +Bridge from Sensu to the ARGO Messaging system + +%files -n argo-sensu-%{name} -f INSTALLED_FILES +%defattr(-,root,root,-) +%config(noreplace) %{_sysconfdir}/%{name}/ams-publisher.conf +%config(noreplace) %{_sysconfdir}/%{name}/metric_data.avsc +%dir %{python3_sitelib}/%{underscore %{name}} +%{python3_sitelib}/%{underscore %{name}}/*.py +%defattr(-,sensu,sensu,-) +%dir %{_localstatedir}/log/%{name}/ +%dir %{_localstatedir}/spool/%{name}/ + + +%package -n argo-nagios-%{name} +Summary: Bridge from Nagios to the ARGO Messaging system + +%description -n argo-nagios-%{name} Bridge from Nagios to the ARGO Messaging system +%files -n argo-nagios-%{name} -f INSTALLED_FILES +%defattr(-,root,root,-) +%config(noreplace) %{_sysconfdir}/%{name}/ams-publisher.conf +%config(noreplace) %{_sysconfdir}/%{name}/metric_data.avsc +%dir %{python3_sitelib}/%{underscore %{name}} +%{python3_sitelib}/%{underscore %{name}}/*.py +%defattr(-,nagios,nagios,-) +%dir %{_localstatedir}/log/%{name}/ +%dir %{_localstatedir}/spool/%{name}/ + + %prep %setup -q @@ -45,16 +79,6 @@ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/%{name}/ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/metrics/ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/alarms/ -%files -f INSTALLED_FILES -%defattr(-,root,root,-) -%config(noreplace) %{_sysconfdir}/%{name}/ams-publisher.conf -%config(noreplace) %{_sysconfdir}/%{name}/metric_data.avsc -%dir %{python3_sitelib}/%{underscore %{name}} -%{python3_sitelib}/%{underscore %{name}}/*.py -%defattr(-,nagios,nagios,-) -%dir %{_localstatedir}/log/%{name}/ -%dir %{_localstatedir}/spool/%{name}/ - %post %systemd_postun_with_restart ams-publisher.service From f893626f2d8a4cdda08ea6dedf9b2edfb9fd5e8c Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 5 May 2022 18:33:23 +0200 Subject: [PATCH 07/32] correct main pkg name --- MANIFEST.in | 2 +- Makefile | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 850eaee..72f94a0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,6 @@ include bin/* include pymod/* include helpers/* include init/* -include argo-nagios-ams-publisher.spec +include ams-publisher.spec recursive-exclude pymodule *.pyc *.pyo diff --git a/Makefile b/Makefile index c81ce68..00efc01 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PKGNAME=argo-nagios-ams-publisher +PKGNAME=ams-publisher SPECFILE=${PKGNAME}.spec PKGVERSION=$(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//') diff --git a/setup.py b/setup.py index 3914ec6..c025f89 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from distutils.core import setup import platform -NAME = 'argo-nagios-ams-publisher' +NAME = 'ams-publisher' def is_c7(): dist = platform.dist() From 9deec89dfe830d973220706051a20b042b022e77 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 08:55:14 +0200 Subject: [PATCH 08/32] refine setup.py for two source builds --- MANIFEST.in | 3 +- ...r.service => ams-publisher-nagios.service} | 0 init/ams-publisher-sensu.service | 15 +++++++ setup.py | 42 +++++++++++++------ 4 files changed, 46 insertions(+), 14 deletions(-) rename init/{ams-publisher.service => ams-publisher-nagios.service} (100%) create mode 100644 init/ams-publisher-sensu.service diff --git a/MANIFEST.in b/MANIFEST.in index 72f94a0..366a9e9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,7 @@ include config/* include bin/* include pymod/* -include helpers/* -include init/* +include init/ams-publisher.service include ams-publisher.spec recursive-exclude pymodule *.pyc *.pyo diff --git a/init/ams-publisher.service b/init/ams-publisher-nagios.service similarity index 100% rename from init/ams-publisher.service rename to init/ams-publisher-nagios.service diff --git a/init/ams-publisher-sensu.service b/init/ams-publisher-sensu.service new file mode 100644 index 0000000..8f82844 --- /dev/null +++ b/init/ams-publisher-sensu.service @@ -0,0 +1,15 @@ +[Unit] +Description=Nagios AMS publisher +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/ams-publisherd +KillMode=process +User=nagios +Group=nagios +SyslogIdentifier=ams-publisher +RuntimeDirectory=argo-nagios-ams-publisher + +[Install] +WantedBy=multi-user.target diff --git a/setup.py b/setup.py index c025f89..eb735b3 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,32 @@ from distutils.core import setup -import platform +from distutils.command.build_py import build_py as _build_py +import shutil +import os + NAME = 'ams-publisher' -def is_c7(): - dist = platform.dist() - for e in dist: - if e.startswith('7'): - return True - return False + +class sensu_build(_build_py): + description='Bridge from Sensu to the ARGO Messaging system', + + def __init__(self, args): + super(_build_py, self).__init__(args) + shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') + self.run_command('build') + self.run_command('sdist') + os.remove('init/ams-publisher.service') + + +class nagios_build(_build_py): + description='Bridge from Nagios to the ARGO Messaging system', + + def __init__(self, args): + super(_build_py, self).__init__(args) + shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') + self.run_command('build') + self.run_command('sdist') + os.remove('init/ams-publisher.service') def get_ver(): @@ -28,13 +46,13 @@ def get_ver(): author='SRCE', author_email='dvrcic@srce.hr', license='ASL 2.0', - description='Bridge from Nagios to the ARGO Messaging system', long_description='Bridge from Nagios to the ARGO Messaging system', url='https://github.com/ARGOeu/argo-nagios-ams-publisher', package_dir={'ams_publisher': 'pymod/'}, packages=['ams_publisher'], - data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), - ('/usr/lib/systemd/system/', ['init/ams-publisher.service']) if is_c7() else \ - ('/etc/init.d/', ['init/ams-publisher'])], scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', - 'bin/ams-publisherd']) + 'bin/ams-publisherd'], + cmdclass={ + 'sensu': sensu_build, 'nagios': nagios_build + } +) From a42c427c4e427c211a545529be98d180590516e2 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 08:57:18 +0200 Subject: [PATCH 09/32] sensu and nagios systemd units --- init/ams-publisher-nagios.service | 2 +- init/ams-publisher-sensu.service | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/init/ams-publisher-nagios.service b/init/ams-publisher-nagios.service index 8f82844..7576501 100644 --- a/init/ams-publisher-nagios.service +++ b/init/ams-publisher-nagios.service @@ -9,7 +9,7 @@ KillMode=process User=nagios Group=nagios SyslogIdentifier=ams-publisher -RuntimeDirectory=argo-nagios-ams-publisher +RuntimeDirectory=ams-publisher [Install] WantedBy=multi-user.target diff --git a/init/ams-publisher-sensu.service b/init/ams-publisher-sensu.service index 8f82844..72078c7 100644 --- a/init/ams-publisher-sensu.service +++ b/init/ams-publisher-sensu.service @@ -1,15 +1,15 @@ [Unit] -Description=Nagios AMS publisher +Description=Sensu AMS publisher After=network.target [Service] Type=simple ExecStart=/usr/bin/ams-publisherd KillMode=process -User=nagios -Group=nagios +User=sensu +Group=sensu SyslogIdentifier=ams-publisher -RuntimeDirectory=argo-nagios-ams-publisher +RuntimeDirectory=ams-publisher [Install] WantedBy=multi-user.target From b8390ea365415c5d4f122ee0f82f4694ccc2c07e Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 09:25:48 +0200 Subject: [PATCH 10/32] remove also build/ dir --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 00efc01..fe53eb1 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ rpm: dist rpmbuild -ta ${PKGNAME}-${PKGVERSION}.tar.gz dist: - rm -rf dist + rm -rf dist build python3 setup.py sdist mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz . rm -rf dist @@ -20,4 +20,5 @@ sources: dist clean: rm -rf ${PKGNAME}-${PKGVERSION}.tar.gz rm -f MANIFEST + rm -rf build rm -rf dist From 69d81df781fc7d20ee2857a04fb7b714e3de9ebf Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 09:30:31 +0200 Subject: [PATCH 11/32] reorganize rpm sections --- ams-publisher.spec | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index 9dba64e..5763327 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -30,6 +30,20 @@ Requires(postun): systemd %description Bridge from Nagios/Sensu to the ARGO Messaging system +%prep +%setup -q + +%build +%{py3_build} + +%install +rm -rf $RPM_BUILD_ROOT +%{py3_install "--record=INSTALLED_FILES"} +install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/%{name}/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/%{name}/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/metrics/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/alarms/ + %package -n argo-sensu-%{name} Summary: Bridge from Sensu to the ARGO Messaging system @@ -65,20 +79,6 @@ Bridge from Nagios to the ARGO Messaging system %dir %{_localstatedir}/spool/%{name}/ -%prep -%setup -q - -%build -%{py3_build} - -%install -rm -rf $RPM_BUILD_ROOT -%{py3_install "--record=INSTALLED_FILES"} -install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/%{name}/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/%{name}/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/metrics/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/alarms/ - %post %systemd_postun_with_restart ams-publisher.service From aa41c62fcd8b4d220aa5cfe535ed70aeca40e6e6 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 09:41:20 +0200 Subject: [PATCH 12/32] call introduced custom commands in distutil from Makefile --- Makefile | 9 ++++++--- setup.py | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index fe53eb1..31e71b1 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,17 @@ rpm: dist dist: rm -rf dist build - python3 setup.py sdist - mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz . + python3 setup.py sensu + mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz + rm -rf dist build + python3 setup.py nagios + mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz rm -rf dist sources: dist clean: - rm -rf ${PKGNAME}-${PKGVERSION}.tar.gz + rm -rf *${PKGNAME}-${PKGVERSION}.tar.gz rm -f MANIFEST rm -rf build rm -rf dist diff --git a/setup.py b/setup.py index eb735b3..7462350 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ class sensu_build(_build_py): def __init__(self, args): super(_build_py, self).__init__(args) - shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') + shutil.copy('init/ams-publisher-sensu.service', 'init/ams-publisher.service') self.run_command('build') self.run_command('sdist') os.remove('init/ams-publisher.service') @@ -50,6 +50,8 @@ def get_ver(): url='https://github.com/ARGOeu/argo-nagios-ams-publisher', package_dir={'ams_publisher': 'pymod/'}, packages=['ams_publisher'], + data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), + ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', 'bin/ams-publisherd'], cmdclass={ From a7977c83c39e4893078aaf9c6f0c53ab2483b6da Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 17:22:39 +0200 Subject: [PATCH 13/32] common spec file --- ams-publisher.spec | 51 ++++++++++++--------------------------- setup.py | 60 ---------------------------------------------- 2 files changed, 15 insertions(+), 96 deletions(-) delete mode 100644 setup.py diff --git a/ams-publisher.spec b/ams-publisher.spec index 5763327..96aa5e0 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -4,7 +4,7 @@ %define stripc() %(echo %1 | sed 's/el7.centos/el7/') %define mydist %{stripc %{dist}} -Name: ams-publisher +Name: SPECNAME-ams-publisher Summary: Bridge from Nagios/Sensu to the ARGO Messaging system Version: 0.3.9 Release: 1%{mydist} @@ -39,45 +39,24 @@ Bridge from Nagios/Sensu to the ARGO Messaging system %install rm -rf $RPM_BUILD_ROOT %{py3_install "--record=INSTALLED_FILES"} -install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/%{name}/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/%{name}/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/metrics/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/%{name}/alarms/ +install --directory --mode 755 %{buildroot}/%{_sysconfdir}/ams-publisher/ +install --directory --mode 755 %{buildroot}/%{_localstatedir}/log/ams-publisher/ +install --directory --mode 755 %{buildroot}/%{_localstatedir}/spool/ams-publisher/metrics/ +install --directory --mode 755 %{buildroot}/%{_localstatedir}/spool/ams-publisher/alarms/ - -%package -n argo-sensu-%{name} -Summary: Bridge from Sensu to the ARGO Messaging system - -%description -n argo-sensu-%{name} +%description Bridge from Sensu to the ARGO Messaging system -%files -n argo-sensu-%{name} -f INSTALLED_FILES +%files -f INSTALLED_FILES %defattr(-,root,root,-) -%config(noreplace) %{_sysconfdir}/%{name}/ams-publisher.conf -%config(noreplace) %{_sysconfdir}/%{name}/metric_data.avsc -%dir %{python3_sitelib}/%{underscore %{name}} -%{python3_sitelib}/%{underscore %{name}}/*.py -%defattr(-,sensu,sensu,-) -%dir %{_localstatedir}/log/%{name}/ -%dir %{_localstatedir}/spool/%{name}/ - - -%package -n argo-nagios-%{name} -Summary: Bridge from Nagios to the ARGO Messaging system - -%description -n argo-nagios-%{name} -Bridge from Nagios to the ARGO Messaging system - -%files -n argo-nagios-%{name} -f INSTALLED_FILES -%defattr(-,root,root,-) -%config(noreplace) %{_sysconfdir}/%{name}/ams-publisher.conf -%config(noreplace) %{_sysconfdir}/%{name}/metric_data.avsc -%dir %{python3_sitelib}/%{underscore %{name}} -%{python3_sitelib}/%{underscore %{name}}/*.py -%defattr(-,nagios,nagios,-) -%dir %{_localstatedir}/log/%{name}/ -%dir %{_localstatedir}/spool/%{name}/ - +%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher.conf +%config(noreplace) %{_sysconfdir}/ams-publisher/metric_data.avsc +%dir %{python3_sitelib}/ams_publisher +%{python3_sitelib}/ams_publisher/*.py +%{_unitdir}/ams-publisher.service +%defattr(-,SPECPERM,SPECPERM,-) +%dir %{_localstatedir}/log/ams-publisher/ +%dir %{_localstatedir}/spool/ams-publisher/ %post %systemd_postun_with_restart ams-publisher.service diff --git a/setup.py b/setup.py deleted file mode 100644 index 7462350..0000000 --- a/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -from distutils.core import setup -from distutils.command.build_py import build_py as _build_py -import shutil -import os - - -NAME = 'ams-publisher' - - -class sensu_build(_build_py): - description='Bridge from Sensu to the ARGO Messaging system', - - def __init__(self, args): - super(_build_py, self).__init__(args) - shutil.copy('init/ams-publisher-sensu.service', 'init/ams-publisher.service') - self.run_command('build') - self.run_command('sdist') - os.remove('init/ams-publisher.service') - - -class nagios_build(_build_py): - description='Bridge from Nagios to the ARGO Messaging system', - - def __init__(self, args): - super(_build_py, self).__init__(args) - shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') - self.run_command('build') - self.run_command('sdist') - os.remove('init/ams-publisher.service') - - -def get_ver(): - try: - with open(f'{NAME}.spec') as f: - for line in f: - if "Version:" in line: - return line.split()[1] - except IOError: - print(f'Make sure that {NAME}.spec is in directory') - raise SystemExit(1) - - -setup( - name=NAME, - version=get_ver(), - author='SRCE', - author_email='dvrcic@srce.hr', - license='ASL 2.0', - long_description='Bridge from Nagios to the ARGO Messaging system', - url='https://github.com/ARGOeu/argo-nagios-ams-publisher', - package_dir={'ams_publisher': 'pymod/'}, - packages=['ams_publisher'], - data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), - ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], - scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', - 'bin/ams-publisherd'], - cmdclass={ - 'sensu': sensu_build, 'nagios': nagios_build - } -) From 828e50420b75ef7da63d023778f8baec3b746681 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 17:23:09 +0200 Subject: [PATCH 14/32] two distutils's setups --- MANIFEST.in | 4 ++- Makefile | 9 +++---- setup.nagios.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ setup.sensu.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 setup.nagios.py create mode 100644 setup.sensu.py diff --git a/MANIFEST.in b/MANIFEST.in index 366a9e9..e771592 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ include config/* include bin/* include pymod/* include init/ams-publisher.service -include ams-publisher.spec +include argo-nagios-ams-publisher.spec +include argo-sensu-ams-publisher.spec +include setup.py recursive-exclude pymodule *.pyc *.pyo diff --git a/Makefile b/Makefile index 31e71b1..74cf1d2 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,11 @@ rpm: dist dist: rm -rf dist build - python3 setup.py sensu - mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz + python3 setup.nagios.py custombuild + mv -f dist/argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz rm -rf dist build - python3 setup.py nagios - mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz - rm -rf dist + python3 setup.sensu.py custombuild + mv -f dist/argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz sources: dist diff --git a/setup.nagios.py b/setup.nagios.py new file mode 100644 index 0000000..433f6ef --- /dev/null +++ b/setup.nagios.py @@ -0,0 +1,64 @@ +from distutils.core import setup +from distutils.command.build_py import build_py as _build_py +import shutil +import os +import glob + + +NAME = 'argo-nagios-ams-publisher' + + +class custom_build(_build_py): + def __init__(self, args): + super(_build_py, self).__init__(args) + shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') + shutil.copy('ams-publisher.spec', 'argo-nagios-ams-publisher.spec') + os.symlink('setup.nagios.py', 'setup.py') + + with open('argo-nagios-ams-publisher.spec', 'r') as specfile: + data = specfile.read() + data = data.replace('SPECNAME-ams-publisher', NAME) + data = data.replace('SPECPERM', 'nagios') + + with open('argo-nagios-ams-publisher.spec', 'w') as specfile: + specfile.write(data) + + self.run_command('build') + self.run_command('sdist') + + os.remove('init/ams-publisher.service') + os.remove('argo-nagios-ams-publisher.spec') + os.remove('setup.py') + + +def get_ver(): + specfile = glob.glob("*.spec") + if specfile: + with open(specfile[0]) as f: + for line in f: + if "Version:" in line: + return line.split()[1] + else: + print(f'Make sure that *.spec is in directory') + raise SystemExit(1) + + +setup( + name=NAME, + version=get_ver(), + author='SRCE', + author_email='dvrcic@srce.hr', + license='ASL 2.0', + description='Bridge from Nagios to the ARGO Messaging system', + long_description='Bridge from Nagios to the ARGO Messaging system', + url='https://github.com/ARGOeu/argo-nagios-ams-publisher', + package_dir={'ams_publisher': 'pymod/'}, + packages=['ams_publisher'], + data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), + ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], + scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', + 'bin/ams-publisherd'], + cmdclass={ + 'custombuild': custom_build + } +) diff --git a/setup.sensu.py b/setup.sensu.py new file mode 100644 index 0000000..ccbd859 --- /dev/null +++ b/setup.sensu.py @@ -0,0 +1,65 @@ +from distutils.core import setup +from distutils.command.build_py import build_py as _build_py +import shutil +import os +import glob + + +NAME = 'argo-sensu-ams-publisher' + + +class custom_build(_build_py): + description='Bridge from Sensu to the ARGO Messaging system', + + def __init__(self, args): + super(_build_py, self).__init__(args) + shutil.copy('init/ams-publisher-sensu.service', 'init/ams-publisher.service') + shutil.copy('ams-publisher.spec', 'argo-sensu-ams-publisher.spec') + os.symlink('setup.sensu.py', 'setup.py') + + with open('argo-sensu-ams-publisher.spec', 'r') as specfile: + data = specfile.read() + data = data.replace('SPECNAME-ams-publisher', NAME) + data = data.replace('SPECPERM', 'sensu') + + with open('argo-sensu-ams-publisher.spec', 'w') as specfile: + specfile.write(data) + + self.run_command('build') + self.run_command('sdist') + + os.remove('init/ams-publisher.service') + os.remove('argo-sensu-ams-publisher.spec') + os.remove('setup.py') + + +def get_ver(): + specfile = glob.glob("*.spec") + if specfile: + with open(specfile[0]) as f: + for line in f: + if "Version:" in line: + return line.split()[1] + else: + print(f'Make sure that *.spec is in directory') + raise SystemExit(1) + + +setup( + name=NAME, + version=get_ver(), + author='SRCE', + author_email='dvrcic@srce.hr', + license='ASL 2.0', + long_description='Bridge from Nagios to the ARGO Messaging system', + url='https://github.com/ARGOeu/argo-nagios-ams-publisher', + package_dir={'ams_publisher': 'pymod/'}, + packages=['ams_publisher'], + data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), + ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], + scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', + 'bin/ams-publisherd'], + cmdclass={ + 'custombuild': custom_build + } +) From ce89a2e07555dfa9310d5817686f37a7bfc58098 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 17:23:56 +0200 Subject: [PATCH 15/32] generic logfile directory reference --- pymod/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymod/log.py b/pymod/log.py index 22cccf1..ce60d6c 100644 --- a/pymod/log.py +++ b/pymod/log.py @@ -4,7 +4,7 @@ import os.path LOGNAME = 'ams-publisher' -LOGFILE = '/var/log/argo-nagios-ams-publisher/ams-publisher.log' +LOGFILE = '/var/log/ams-publisher/ams-publisher.log' class Logger(object): From 7b28a46ea8710b81a88ffecc037c165b2b8be15d Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 6 May 2022 17:27:27 +0200 Subject: [PATCH 16/32] build RPMs from both source tarballs --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 74cf1d2..cc82734 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ SPECFILE=${PKGNAME}.spec PKGVERSION=$(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//') srpm: dist - rpmbuild -ts ${PKGNAME}-${PKGVERSION}.tar.gz + rpmbuild -ts *.tar.gz rpm: dist - rpmbuild -ta ${PKGNAME}-${PKGVERSION}.tar.gz + rpmbuild -ta *.tar.gz dist: rm -rf dist build From c95fcc936e777ec77c1c7d1a298e1c1c52ce60bf Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 9 May 2022 16:04:51 +0200 Subject: [PATCH 17/32] create source tarball with files for both packages and have two package definitions in spec --- MANIFEST.in | 6 +- Makefile | 17 ++-- ams-publisher.spec | 79 ++++++++++++++----- ...blisher.conf => ams-publisher-nagios.conf} | 0 config/ams-publisher-sensu.conf | 51 ++++++++++++ setup.py | 35 ++++++++ 6 files changed, 155 insertions(+), 33 deletions(-) rename config/{ams-publisher.conf => ams-publisher-nagios.conf} (100%) create mode 100644 config/ams-publisher-sensu.conf create mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index e771592..493eed0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,9 @@ include config/* include bin/* include pymod/* -include init/ams-publisher.service -include argo-nagios-ams-publisher.spec -include argo-sensu-ams-publisher.spec +include init/ams-publisher-nagios.service +include init/ams-publisher-sensu.service +include ams-publisher.spec include setup.py recursive-exclude pymodule *.pyc *.pyo diff --git a/Makefile b/Makefile index cc82734..00efc01 100644 --- a/Makefile +++ b/Makefile @@ -4,23 +4,20 @@ SPECFILE=${PKGNAME}.spec PKGVERSION=$(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//') srpm: dist - rpmbuild -ts *.tar.gz + rpmbuild -ts ${PKGNAME}-${PKGVERSION}.tar.gz rpm: dist - rpmbuild -ta *.tar.gz + rpmbuild -ta ${PKGNAME}-${PKGVERSION}.tar.gz dist: - rm -rf dist build - python3 setup.nagios.py custombuild - mv -f dist/argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz argo-nagios-${PKGNAME}-${PKGVERSION}.tar.gz - rm -rf dist build - python3 setup.sensu.py custombuild - mv -f dist/argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz argo-sensu-${PKGNAME}-${PKGVERSION}.tar.gz + rm -rf dist + python3 setup.py sdist + mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz . + rm -rf dist sources: dist clean: - rm -rf *${PKGNAME}-${PKGVERSION}.tar.gz + rm -rf ${PKGNAME}-${PKGVERSION}.tar.gz rm -f MANIFEST - rm -rf build rm -rf dist diff --git a/ams-publisher.spec b/ams-publisher.spec index 96aa5e0..ed2fd77 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -4,8 +4,8 @@ %define stripc() %(echo %1 | sed 's/el7.centos/el7/') %define mydist %{stripc %{dist}} -Name: SPECNAME-ams-publisher -Summary: Bridge from Nagios/Sensu to the ARGO Messaging system +Name: ams-publisher +Summary: Bridge from Sensu/Nagios to the ARGO Messaging system Version: 0.3.9 Release: 1%{mydist} @@ -27,46 +27,85 @@ Requires(post): systemd Requires(preun): systemd Requires(postun): systemd -%description -Bridge from Nagios/Sensu to the ARGO Messaging system - %prep %setup -q +%description +Bridge from Sensu/Nagios to the ARGO Messaging system + %build %{py3_build} %install rm -rf $RPM_BUILD_ROOT -%{py3_install "--record=INSTALLED_FILES"} -install --directory --mode 755 %{buildroot}/%{_sysconfdir}/ams-publisher/ -install --directory --mode 755 %{buildroot}/%{_localstatedir}/log/ams-publisher/ -install --directory --mode 755 %{buildroot}/%{_localstatedir}/spool/ams-publisher/metrics/ -install --directory --mode 755 %{buildroot}/%{_localstatedir}/spool/ams-publisher/alarms/ +%{py3_install} +install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/ams-publisher/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/ams-publisher/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/metrics/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/alarms/ -%description + +%package -n argo-nagios-ams-publisher +Summary: Bridge from Nagios to the ARGO Messaging system + +%description -n argo-nagios-ams-publisher +Bridge from Nagios to the ARGO Messaging system + +%files -n argo-nagios-ams-publisher +%defattr(-,root,root,-) +%{_bindir}/ams-alarm-to-queue +%{_bindir}/ams-metric-to-queue +%{_bindir}/ams-publisherd +%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher-nagios.conf +%config(noreplace) %{_sysconfdir}/ams-publisher/metric_data.avsc +%dir %{python3_sitelib}/ams_publisher +%{python3_sitelib}/ams_publisher/*.py +%{python3_sitelib}/ams_publisher/__pycache__/ +%{python3_sitelib}/*.egg-info +%{_unitdir}/ams-publisher-nagios.service +%defattr(-,nagios,nagios,-) +%dir %{_localstatedir}/log/ams-publisher/ +%dir %{_localstatedir}/spool/ams-publisher/ + +%post -n argo-nagios-ams-publisher +%systemd_postun_with_restart ams-publisher-nagios.service + +%preun -n argo-nagios-ams-publisher +%systemd_preun ams-publisher.service + + +%package -n argo-sensu-ams-publisher +Summary: Bridge from Sensu to the ARGO Messaging system + +%description -n argo-sensu-ams-publisher Bridge from Sensu to the ARGO Messaging system -%files -f INSTALLED_FILES +%files -n argo-sensu-ams-publisher %defattr(-,root,root,-) -%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher.conf +%{_bindir}/ams-alarm-to-queue +%{_bindir}/ams-metric-to-queue +%{_bindir}/ams-publisherd +%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher-sensu.conf %config(noreplace) %{_sysconfdir}/ams-publisher/metric_data.avsc %dir %{python3_sitelib}/ams_publisher %{python3_sitelib}/ams_publisher/*.py -%{_unitdir}/ams-publisher.service -%defattr(-,SPECPERM,SPECPERM,-) +%{python3_sitelib}/ams_publisher/__pycache__/ +%{python3_sitelib}/*.egg-info +%{_unitdir}/ams-publisher-sensu.service +%defattr(-,sensu,sensu,-) %dir %{_localstatedir}/log/ams-publisher/ %dir %{_localstatedir}/spool/ams-publisher/ -%post -%systemd_postun_with_restart ams-publisher.service +%post -n argo-sensu-ams-publisher +%systemd_postun_with_restart ams-publisher-nagios.service + +%preun -n argo-sensu-ams-publisher +%systemd_preun ams-publisher.service + %clean rm -rf $RPM_BUILD_ROOT -%preun -%systemd_preun ams-publisher.service - %changelog * Mon Feb 1 2021 Daniel Vrcic - 0.3.9-1%{?dist} - ARGO-2855 ams-publisher py3 switch diff --git a/config/ams-publisher.conf b/config/ams-publisher-nagios.conf similarity index 100% rename from config/ams-publisher.conf rename to config/ams-publisher-nagios.conf diff --git a/config/ams-publisher-sensu.conf b/config/ams-publisher-sensu.conf new file mode 100644 index 0000000..a0694b2 --- /dev/null +++ b/config/ams-publisher-sensu.conf @@ -0,0 +1,51 @@ +[General] +Host = nagioshost +RunAsUser = nagios +StatsEveryHour = 24 +PublishMsgFile = False +PublishMsgFileDir = /published +PublishArgoMessaging = True +TimeZone = UTC + +[Queue_Metrics] +Directory = /var/spool/argo-nagios-ams-publisher/metrics/ +Rate = 10 +Purge = True +PurgeEverySec = 300 +MaxTemp = 300 +MaxLock = 0 +Granularity = 60 + +[Topic_Metrics] +Host = messaging-devel.argo.grnet.gr +Key = EGIKEY +Project = EGI +Bulksize = 100 +Topic = metric_data +MsgType = metric_data +Avro = True +AvroSchema = /etc/ams-publisher/metric_data.avsc +Retry = 5 +Timeout = 60 +SleepRetry = 300 + +[Queue_Alarms] +Directory = /var/spool/argo-nagios-ams-publisher/alarms/ +Rate = 10 +Purge = True +PurgeEverySec = 300 +MaxTemp = 300 +MaxLock = 0 +Granularity = 60 + +[Topic_Alarms] +Host = messaging-devel.argo.grnet.gr +Key = EGIKEY +Project = EGI +Bulksize = 100 +Topic = alarms +MsgType = alarm +Avro = False +Retry = 5 +Timeout = 60 +SleepRetry = 300 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5b1931a --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from distutils.core import setup +import platform + +NAME = 'ams-publisher' + + +def get_ver(): + try: + with open(f'{NAME}.spec') as f: + for line in f: + if "Version:" in line: + return line.split()[1] + except IOError: + print(f'Make sure that {NAME}.spec is in directory') + raise SystemExit(1) + + +setup( + name=NAME, + version=get_ver(), + author='SRCE', + author_email='dvrcic@srce.hr', + license='ASL 2.0', + description='Bridge from Nagios/Sensu to the ARGO Messaging system', + long_description='Bridge from Nagios/Sensu to the ARGO Messaging system', + url='https://github.com/ARGOeu/argo-nagios-ams-publisher', + package_dir={'ams_publisher': 'pymod/'}, + packages=['ams_publisher'], + data_files=[('/etc/ams-publisher/', ['config/ams-publisher-sensu.conf', + 'config/ams-publisher-nagios.conf', + 'config/metric_data.avsc']), + ('/usr/lib/systemd/system/', ['init/ams-publisher-nagios.service', + 'init/ams-publisher-sensu.service'])], + scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', + 'bin/ams-publisherd']) From aef8726dbdbfef55fbc165db0f02c28204879191 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 9 May 2022 16:21:10 +0200 Subject: [PATCH 18/32] correct unit file reference in scriptlets --- ams-publisher.spec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index ed2fd77..a49d9ee 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -71,7 +71,7 @@ Bridge from Nagios to the ARGO Messaging system %systemd_postun_with_restart ams-publisher-nagios.service %preun -n argo-nagios-ams-publisher -%systemd_preun ams-publisher.service +%systemd_preun ams-publisher-nagios.service %package -n argo-sensu-ams-publisher @@ -97,10 +97,10 @@ Bridge from Sensu to the ARGO Messaging system %dir %{_localstatedir}/spool/ams-publisher/ %post -n argo-sensu-ams-publisher -%systemd_postun_with_restart ams-publisher-nagios.service +%systemd_postun_with_restart ams-publisher-sensu.service %preun -n argo-sensu-ams-publisher -%systemd_preun ams-publisher.service +%systemd_preun ams-publisher-sensu.service %clean From e84e37f6890bcd33b62d35fb9a01b634f6883bb5 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 9 May 2022 16:24:30 +0200 Subject: [PATCH 19/32] delete two setup.py's --- setup.nagios.py | 64 ------------------------------------------------ setup.sensu.py | 65 ------------------------------------------------- 2 files changed, 129 deletions(-) delete mode 100644 setup.nagios.py delete mode 100644 setup.sensu.py diff --git a/setup.nagios.py b/setup.nagios.py deleted file mode 100644 index 433f6ef..0000000 --- a/setup.nagios.py +++ /dev/null @@ -1,64 +0,0 @@ -from distutils.core import setup -from distutils.command.build_py import build_py as _build_py -import shutil -import os -import glob - - -NAME = 'argo-nagios-ams-publisher' - - -class custom_build(_build_py): - def __init__(self, args): - super(_build_py, self).__init__(args) - shutil.copy('init/ams-publisher-nagios.service', 'init/ams-publisher.service') - shutil.copy('ams-publisher.spec', 'argo-nagios-ams-publisher.spec') - os.symlink('setup.nagios.py', 'setup.py') - - with open('argo-nagios-ams-publisher.spec', 'r') as specfile: - data = specfile.read() - data = data.replace('SPECNAME-ams-publisher', NAME) - data = data.replace('SPECPERM', 'nagios') - - with open('argo-nagios-ams-publisher.spec', 'w') as specfile: - specfile.write(data) - - self.run_command('build') - self.run_command('sdist') - - os.remove('init/ams-publisher.service') - os.remove('argo-nagios-ams-publisher.spec') - os.remove('setup.py') - - -def get_ver(): - specfile = glob.glob("*.spec") - if specfile: - with open(specfile[0]) as f: - for line in f: - if "Version:" in line: - return line.split()[1] - else: - print(f'Make sure that *.spec is in directory') - raise SystemExit(1) - - -setup( - name=NAME, - version=get_ver(), - author='SRCE', - author_email='dvrcic@srce.hr', - license='ASL 2.0', - description='Bridge from Nagios to the ARGO Messaging system', - long_description='Bridge from Nagios to the ARGO Messaging system', - url='https://github.com/ARGOeu/argo-nagios-ams-publisher', - package_dir={'ams_publisher': 'pymod/'}, - packages=['ams_publisher'], - data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), - ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], - scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', - 'bin/ams-publisherd'], - cmdclass={ - 'custombuild': custom_build - } -) diff --git a/setup.sensu.py b/setup.sensu.py deleted file mode 100644 index ccbd859..0000000 --- a/setup.sensu.py +++ /dev/null @@ -1,65 +0,0 @@ -from distutils.core import setup -from distutils.command.build_py import build_py as _build_py -import shutil -import os -import glob - - -NAME = 'argo-sensu-ams-publisher' - - -class custom_build(_build_py): - description='Bridge from Sensu to the ARGO Messaging system', - - def __init__(self, args): - super(_build_py, self).__init__(args) - shutil.copy('init/ams-publisher-sensu.service', 'init/ams-publisher.service') - shutil.copy('ams-publisher.spec', 'argo-sensu-ams-publisher.spec') - os.symlink('setup.sensu.py', 'setup.py') - - with open('argo-sensu-ams-publisher.spec', 'r') as specfile: - data = specfile.read() - data = data.replace('SPECNAME-ams-publisher', NAME) - data = data.replace('SPECPERM', 'sensu') - - with open('argo-sensu-ams-publisher.spec', 'w') as specfile: - specfile.write(data) - - self.run_command('build') - self.run_command('sdist') - - os.remove('init/ams-publisher.service') - os.remove('argo-sensu-ams-publisher.spec') - os.remove('setup.py') - - -def get_ver(): - specfile = glob.glob("*.spec") - if specfile: - with open(specfile[0]) as f: - for line in f: - if "Version:" in line: - return line.split()[1] - else: - print(f'Make sure that *.spec is in directory') - raise SystemExit(1) - - -setup( - name=NAME, - version=get_ver(), - author='SRCE', - author_email='dvrcic@srce.hr', - license='ASL 2.0', - long_description='Bridge from Nagios to the ARGO Messaging system', - url='https://github.com/ARGOeu/argo-nagios-ams-publisher', - package_dir={'ams_publisher': 'pymod/'}, - packages=['ams_publisher'], - data_files=[('/etc/ams-publisher/', ['config/ams-publisher.conf', 'config/metric_data.avsc']), - ('/usr/lib/systemd/system/', ['init/ams-publisher.service'])], - scripts=['bin/ams-alarm-to-queue', 'bin/ams-metric-to-queue', - 'bin/ams-publisherd'], - cmdclass={ - 'custombuild': custom_build - } -) From 4f7dbd40d9ea77b7f4c631dc207119608d3028c4 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 9 May 2022 16:54:51 +0200 Subject: [PATCH 20/32] change runas user and cache directory paths in config templates --- config/ams-publisher-nagios.conf | 4 ++-- config/ams-publisher-sensu.conf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/ams-publisher-nagios.conf b/config/ams-publisher-nagios.conf index a0694b2..9dd2499 100644 --- a/config/ams-publisher-nagios.conf +++ b/config/ams-publisher-nagios.conf @@ -8,7 +8,7 @@ PublishArgoMessaging = True TimeZone = UTC [Queue_Metrics] -Directory = /var/spool/argo-nagios-ams-publisher/metrics/ +Directory = /var/spool/ams-publisher/metrics/ Rate = 10 Purge = True PurgeEverySec = 300 @@ -30,7 +30,7 @@ Timeout = 60 SleepRetry = 300 [Queue_Alarms] -Directory = /var/spool/argo-nagios-ams-publisher/alarms/ +Directory = /var/spool/ams-publisher/alarms/ Rate = 10 Purge = True PurgeEverySec = 300 diff --git a/config/ams-publisher-sensu.conf b/config/ams-publisher-sensu.conf index a0694b2..f6398e1 100644 --- a/config/ams-publisher-sensu.conf +++ b/config/ams-publisher-sensu.conf @@ -1,6 +1,6 @@ [General] Host = nagioshost -RunAsUser = nagios +RunAsUser = sensu StatsEveryHour = 24 PublishMsgFile = False PublishMsgFileDir = /published @@ -8,7 +8,7 @@ PublishArgoMessaging = True TimeZone = UTC [Queue_Metrics] -Directory = /var/spool/argo-nagios-ams-publisher/metrics/ +Directory = /var/spool/ams-publisher/metrics/ Rate = 10 Purge = True PurgeEverySec = 300 @@ -30,7 +30,7 @@ Timeout = 60 SleepRetry = 300 [Queue_Alarms] -Directory = /var/spool/argo-nagios-ams-publisher/alarms/ +Directory = /var/spool/ams-publisher/alarms/ Rate = 10 Purge = True PurgeEverySec = 300 From 8ae2fa08053ecfc2398a52280b7ee29c61c29649 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 9 May 2022 23:12:12 +0200 Subject: [PATCH 21/32] load first ams-publisher*.conf found in /etc --- pymod/config.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pymod/config.py b/pymod/config.py index 26cc01a..226089b 100644 --- a/pymod/config.py +++ b/pymod/config.py @@ -1,8 +1,22 @@ import configparser import sys +import glob + from pytz import timezone, UnknownTimeZoneError -conf = '/etc/ams-publisher/ams-publisher.conf' + +def get_conffile(logger=None): + confwildcard = '/etc/ams-publisher/ams-publisher*.conf' + matches = glob.glob(confwildcard) + if matches: + return matches[0] + else: + if logger: + logger.error('Missing config file, pattern %s not found' % confwildcard) + raise SystemExit(1) + else: + sys.stderr.write('Missing config file, pattern %s not found\n' % confwildcard) + raise SystemExit(1) def get_queue_granul(queue): @@ -28,6 +42,8 @@ def parse_config(logger=None): reqsections = set(['queue_', 'topic_', 'general']) confopts = dict() + conf = get_conffile(logger) + try: config = configparser.ConfigParser() if config.read(conf): From a2add7fbb097889cde7f34de8781ebe4dfd4bf2f Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Tue, 10 May 2022 00:07:39 +0200 Subject: [PATCH 22/32] create just main spool directory --- ams-publisher.spec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index a49d9ee..2bffda3 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -41,8 +41,7 @@ rm -rf $RPM_BUILD_ROOT %{py3_install} install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/ams-publisher/ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/ams-publisher/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/metrics/ -install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/alarms/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/ %package -n argo-nagios-ams-publisher From c4329fc9fc98563be211dc7232375c2617007cd9 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Tue, 10 May 2022 00:19:20 +0200 Subject: [PATCH 23/32] have conflicts defined --- ams-publisher.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ams-publisher.spec b/ams-publisher.spec index 2bffda3..0974358 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -46,6 +46,7 @@ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publi %package -n argo-nagios-ams-publisher Summary: Bridge from Nagios to the ARGO Messaging system +Conflicts: argo-sensu-ams-publisher %description -n argo-nagios-ams-publisher Bridge from Nagios to the ARGO Messaging system @@ -75,6 +76,7 @@ Bridge from Nagios to the ARGO Messaging system %package -n argo-sensu-ams-publisher Summary: Bridge from Sensu to the ARGO Messaging system +Conflicts: argo-nagios-ams-publisher %description -n argo-sensu-ams-publisher Bridge from Sensu to the ARGO Messaging system From f677495af50e7e64000023c5c3acf8982ae19a2a Mon Sep 17 00:00:00 2001 From: Themis Zamani Date: Tue, 10 May 2022 10:08:31 +0300 Subject: [PATCH 24/32] Update Jenkinsfile --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1a3c021..854a30b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,10 @@ pipeline { agent any options { - checkoutToSubdirectory('argo-nagios-ams-publisher') + checkoutToSubdirectory('ams-publisher') } environment { - PROJECT_DIR="argo-nagios-ams-publisher" + PROJECT_DIR="ams-publisher" GIT_COMMIT=sh(script: "cd ${WORKSPACE}/$PROJECT_DIR && git log -1 --format=\"%H\"",returnStdout: true).trim() GIT_COMMIT_HASH=sh(script: "cd ${WORKSPACE}/$PROJECT_DIR && git log -1 --format=\"%H\" | cut -c1-7",returnStdout: true).trim() GIT_COMMIT_DATE=sh(script: "date -d \"\$(cd ${WORKSPACE}/$PROJECT_DIR && git show -s --format=%ci ${GIT_COMMIT_HASH})\" \"+%Y%m%d%H%M%S\"",returnStdout: true).trim() From c911c3f6afaf4e951ef4abb6c18893a68bfadd93 Mon Sep 17 00:00:00 2001 From: Themis Zamani Date: Tue, 10 May 2022 10:09:27 +0300 Subject: [PATCH 25/32] Update and rename argo-nagios-ams-publisher.spec to ams-publisher.spec --- argo-nagios-ams-publisher.spec => ams-publisher.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename argo-nagios-ams-publisher.spec => ams-publisher.spec (98%) diff --git a/argo-nagios-ams-publisher.spec b/ams-publisher.spec similarity index 98% rename from argo-nagios-ams-publisher.spec rename to ams-publisher.spec index c31f5de..dfa0a96 100644 --- a/argo-nagios-ams-publisher.spec +++ b/ams-publisher.spec @@ -4,14 +4,14 @@ %define stripc() %(echo %1 | sed 's/el7.centos/el7/') %define mydist %{stripc %{dist}} -Name: argo-nagios-ams-publisher +Name: ams-publisher Version: 0.3.9 Release: 1%{mydist} Summary: Bridge from Nagios to the ARGO Messaging system Group: Network/Monitoring License: ASL 2.0 -URL: https://github.com/ARGOeu/argo-nagios-ams-publisher +URL: https://github.com/ARGOeu/ams-publisher Source0: %{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) From cdccfa4b382a999c74321e9395d5cff962e0f714 Mon Sep 17 00:00:00 2001 From: Themis Zamani Date: Tue, 10 May 2022 10:12:07 +0300 Subject: [PATCH 26/32] Update README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b8ecc2a..8160079 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -# argo-nagios-ams-publisher +# ams-publisher ## Description -`argo-nagios-ams-publisher` is a component acting as bridge from Nagios to ARGO Messaging system. It's essential part of software stack running on ARGO monitoring instance and is responsible for forming and dispatching messages that wrap up results of Nagios probes/tests. It is running as a unix daemon and it consists of two subsystems: +`ams-publisher` is a component acting as bridge from Nagios to ARGO Messaging system. It's essential part of software stack running on ARGO monitoring instance and is responsible for forming and dispatching messages that wrap up results of Nagios probes/tests. It is running as a unix daemon and it consists of two subsystems: - queueing mechanism - publishing/dispatching part -Messages are cached in local directory queue with the help of OCSP Nagios commands and each queue is being monitored and consumed by the daemon. After configurable amount of accumulated messages, publisher that is associated to queue sends them to ARGO Messaging system and drains the queue. `argo-nagios-ams-publisher` is written in multiprocessing manner so there is support for multiple (consume, publish) pairs where for each, new worker process will be spawned. +Messages are cached in local directory queue with the help of OCSP Nagios commands and each queue is being monitored and consumed by the daemon. After configurable amount of accumulated messages, publisher that is associated to queue sends them to ARGO Messaging system and drains the queue. `ams-publisher` is written in multiprocessing manner so there is support for multiple (consume, publish) pairs where for each, new worker process will be spawned. -Filling and draining of directory queue is asynchronous. Nagios delivers results on its own constant rate while `argo-nagios-ams-publisher` consume and publish them on its own configurable constant rate. It's important to keep the two rates close enough so that the results don't pile up in the queue and leave it early. Component has a mechanism of inspection of rates and trends over time to keep the constants in sync. Also it's resilient to network issues so it will retry configurable number of times to send a messages to ARGO Messaging system. It's also important to note that consume and publish of the queue is a serial process so if publish is stopped, consume part of the worker will be also stopped. That could lead to pile up of results in the queue and since every result is represented as a one file on the file system, easily exhaustion of free inodes and therefore unusable monitoring instance. +Filling and draining of directory queue is asynchronous. Nagios delivers results on its own constant rate while `ams-publisher` consume and publish them on its own configurable constant rate. It's important to keep the two rates close enough so that the results don't pile up in the queue and leave it early. Component has a mechanism of inspection of rates and trends over time to keep the constants in sync. Also it's resilient to network issues so it will retry configurable number of times to send a messages to ARGO Messaging system. It's also important to note that consume and publish of the queue is a serial process so if publish is stopped, consume part of the worker will be also stopped. That could lead to pile up of results in the queue and since every result is represented as a one file on the file system, easily exhaustion of free inodes and therefore unusable monitoring instance. More about [Directory queue design](https://dirq.readthedocs.io/en/latest/queuesimple.html#directory-structure) @@ -29,7 +29,7 @@ Complete list of features are: Component is supported on CentOS 6 and CentOS 7. RPM packages and all needed dependencies are available in ARGO repositories so installation of component simply narrows down to installing a package: - yum install -y argo-nagios-ams-publisher + yum install -y ams-publisher Component relies on: - `argo-ams-library` - interaction with ARGO Messaging @@ -42,14 +42,14 @@ Component relies on: | File Types | Destination | |-------------------|----------------------------------------------------| -| Configuration | `/etc/argo-nagios-ams-publisher/ams-publisher.conf`| +| Configuration | `/etc/ams-publisher/ams-publisher.conf`| | Daemon component | `/usr/bin/ams-publisherd` | | Cache delivery | `/usr/bin/ams-alarm-to-queue, ams-metric-to-queue` | | Init script (C6) | `/etc/init.d/ams-publisher` | | SystemD Unit (C7) | `/usr/lib/systemd/system/ams-publisher.service` | -| Local caches | `/var/spool/argo-nagios-ams-publisher/` | -| Inspection socket | `/var/run/argo-nagios-ams-publisher/sock` | -| Log files | `/var/log/argo-nagios-ams-publisher/` | +| Local caches | `/var/spool/ams-publisher/` | +| Inspection socket | `/var/run/ams-publisher/sock` | +| Log files | `/var/log/ams-publisher/` | ## Configuration @@ -66,7 +66,7 @@ PublishMsgFile = False PublishMsgFileDir = /published PublishArgoMessaging = True TimeZone = UTC -StatSocket = /var/run/argo-nagios-ams-publisher/sock +StatSocket = /var/run/ams-publisher/sock ``` * `Host` - FQDN of ARGO Monitoring instance that will be part of formed messages dispatched to ARGO Messaging system @@ -83,7 +83,7 @@ Eachs `(queue, topic)` section pair designates one worker process. Two sections Example of one such pair: ``` [Queue_Metrics] -Directory = /var/spool/argo-nagios-ams-publisher/metrics/ +Directory = /var/spool/ams-publisher/metrics/ Rate = 10 Purge = True PurgeEverySec = 300 @@ -99,7 +99,7 @@ Topic = metric_data Bulksize = 100 MsgType = metric_data Avro = True -AvroSchema = /etc/argo-nagios-ams-publisher/metric_data.avsc +AvroSchema = /etc/ams-publisher/metric_data.avsc Retry = 5 Timeout = 60 SleepRetry = 300 From ad03c60ae3f039edb2d8dcca0a920f508113daed Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Tue, 10 May 2022 13:10:13 +0200 Subject: [PATCH 27/32] add dependencies on nagios and sensu-go-backend --- ams-publisher.spec | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index 0974358..6ebbb27 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -45,8 +45,9 @@ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publi %package -n argo-nagios-ams-publisher -Summary: Bridge from Nagios to the ARGO Messaging system +Summary: Bridge from Nagios to the ARGO Messaging system Conflicts: argo-sensu-ams-publisher +Requires: nagios %description -n argo-nagios-ams-publisher Bridge from Nagios to the ARGO Messaging system @@ -75,8 +76,9 @@ Bridge from Nagios to the ARGO Messaging system %package -n argo-sensu-ams-publisher -Summary: Bridge from Sensu to the ARGO Messaging system +Summary: Bridge from Sensu to the ARGO Messaging system Conflicts: argo-nagios-ams-publisher +Requires: sensu-go-backend %description -n argo-sensu-ams-publisher Bridge from Sensu to the ARGO Messaging system From 623007d8dea91d9aeef05a748ed3cc19431cb97d Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Tue, 10 May 2022 13:20:26 +0200 Subject: [PATCH 28/32] re-add ams-publisher.spec --- ams-publisher.spec | 170 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/ams-publisher.spec b/ams-publisher.spec index e69de29..6ebbb27 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -0,0 +1,170 @@ +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} + +%define underscore() %(echo %1 | sed 's/-/_/g') +%define stripc() %(echo %1 | sed 's/el7.centos/el7/') +%define mydist %{stripc %{dist}} + +Name: ams-publisher +Summary: Bridge from Sensu/Nagios to the ARGO Messaging system +Version: 0.3.9 +Release: 1%{mydist} + +Group: Network/Monitoring +License: ASL 2.0 +URL: https://github.com/ARGOeu/argo-nagios-ams-publisher +Source0: %{name}-%{version}.tar.gz + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildArch: noarch +BuildRequires: python3-devel +Requires: python3-argo-ams-library +Requires: python3-avro +Requires: python3-dirq +Requires: python3-messaging +Requires: python36-pytz + +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd + +%prep +%setup -q + +%description +Bridge from Sensu/Nagios to the ARGO Messaging system + +%build +%{py3_build} + +%install +rm -rf $RPM_BUILD_ROOT +%{py3_install} +install --directory --mode 755 $RPM_BUILD_ROOT/%{_sysconfdir}/ams-publisher/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/log/ams-publisher/ +install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publisher/ + + +%package -n argo-nagios-ams-publisher +Summary: Bridge from Nagios to the ARGO Messaging system +Conflicts: argo-sensu-ams-publisher +Requires: nagios + +%description -n argo-nagios-ams-publisher +Bridge from Nagios to the ARGO Messaging system + +%files -n argo-nagios-ams-publisher +%defattr(-,root,root,-) +%{_bindir}/ams-alarm-to-queue +%{_bindir}/ams-metric-to-queue +%{_bindir}/ams-publisherd +%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher-nagios.conf +%config(noreplace) %{_sysconfdir}/ams-publisher/metric_data.avsc +%dir %{python3_sitelib}/ams_publisher +%{python3_sitelib}/ams_publisher/*.py +%{python3_sitelib}/ams_publisher/__pycache__/ +%{python3_sitelib}/*.egg-info +%{_unitdir}/ams-publisher-nagios.service +%defattr(-,nagios,nagios,-) +%dir %{_localstatedir}/log/ams-publisher/ +%dir %{_localstatedir}/spool/ams-publisher/ + +%post -n argo-nagios-ams-publisher +%systemd_postun_with_restart ams-publisher-nagios.service + +%preun -n argo-nagios-ams-publisher +%systemd_preun ams-publisher-nagios.service + + +%package -n argo-sensu-ams-publisher +Summary: Bridge from Sensu to the ARGO Messaging system +Conflicts: argo-nagios-ams-publisher +Requires: sensu-go-backend + +%description -n argo-sensu-ams-publisher +Bridge from Sensu to the ARGO Messaging system + +%files -n argo-sensu-ams-publisher +%defattr(-,root,root,-) +%{_bindir}/ams-alarm-to-queue +%{_bindir}/ams-metric-to-queue +%{_bindir}/ams-publisherd +%config(noreplace) %{_sysconfdir}/ams-publisher/ams-publisher-sensu.conf +%config(noreplace) %{_sysconfdir}/ams-publisher/metric_data.avsc +%dir %{python3_sitelib}/ams_publisher +%{python3_sitelib}/ams_publisher/*.py +%{python3_sitelib}/ams_publisher/__pycache__/ +%{python3_sitelib}/*.egg-info +%{_unitdir}/ams-publisher-sensu.service +%defattr(-,sensu,sensu,-) +%dir %{_localstatedir}/log/ams-publisher/ +%dir %{_localstatedir}/spool/ams-publisher/ + +%post -n argo-sensu-ams-publisher +%systemd_postun_with_restart ams-publisher-sensu.service + +%preun -n argo-sensu-ams-publisher +%systemd_preun ams-publisher-sensu.service + + +%clean +rm -rf $RPM_BUILD_ROOT + +%changelog +* Mon Feb 1 2021 Daniel Vrcic - 0.3.9-1%{?dist} +- ARGO-2855 ams-publisher py3 switch +- ARGO-2929 Let systemd handle runtime directory +* Thu Oct 8 2020 Daniel Vrcic - 0.3.8-1%{?dist} +- remove leftovers from erroneous SIGHUP handling +* Wed Jul 8 2020 Daniel Vrcic - 0.3.7-1%{?dist} +- ARGO-2378 RPM post install should restart service not stop it +- ARGO-844 Complete README for ams-publisher +* Tue Mar 31 2020 Daniel Vrcic - 0.3.6-1%{?dist} +- ARGO-2224 Local delivery cache tool should pass non-ascii characters +* Tue Apr 23 2019 Daniel Vrcic - 0.3.5-2%{?dist} +- regression fix to include site name in `site` field of notification +* Wed Apr 17 2019 Daniel Vrcic - 0.3.5-1%{?dist} +- ARGO-1726 Pass site name in metric results +* Wed Mar 6 2019 Daniel Vrcic - 0.3.4-1%{?dist} +- verbose log messages for errors not handled in argo-ams-library +* Tue Feb 5 2019 Daniel Vrcic - 0.3.3-1%{?dist} +- ARGO-1624 Catch all exceptions and warnings from AMS +* Thu Nov 8 2018 Daniel Vrcic - 0.3.2-1%{?dist} +- ARGO-1429 Improved msg counter stats for probe testing purposes +- ARGO-1408 Ensure correct permissions on pidfile directory +- ARGO-1348 Descriptive error in case delivery cache tool is called with queue + path not specified in configs +* Tue Jun 19 2018 Daniel Vrcic - 0.3.1-1%{?dist} +- ARGO-1250 Inspection local socket is left with root permissions +- ARGO-1147 AMS publisher to add optional field +- ARGO-986 Purger should not try to remove non-existing cache msg +* Tue Mar 27 2018 Daniel Vrcic - 0.3.0-1%{?dist} +- ARGO-1084 Connection settings per topic publisher +- ARGO-1023 Send messages to prod and devel AMS instance in parallel +- ARGO-1055 Last time stats report not updated +- ARGO-1051 Ensure service stop called on system shutdown +- ARGO-1004 UTC timestamp instead of localtime for dispatched results +- ARGO-978 Add systemd init script +- ARGO-806 AMS Publisher nagios testing method for upcoming probe +* Wed Dec 20 2017 Daniel Vrcic - 0.2.1-1%{?dist} +- Centos 7 code fixes and spec update +- ARGO-930 Service stop should not depend on successful config parse +- ARGO-700 Delay each msg publish try in case of connection problems +* Mon Jun 5 2017 Daniel Vrcic - 0.2.0-1%{?dist} +- ARGO-797 argo-nagios-ams-publisher overwrites configuration +- ARGO-802 Singleton config object with shared config options +- ARGO-803 Refactor publisher class +- ARGO-815 Sent number of messages reported incorrectly +- ARGO-826 Pick up only HARD states +* Thu Apr 27 2017 Daniel Vrcic - 0.1.3-1%{?dist} +- ARGO-788 Skip empty files/messages +* Thu Mar 30 2017 Daniel Vrcic - 0.1.2-1%{?dist} +- ARGO-764 Add url_history and url_help message fields for alarms +* Tue Mar 14 2017 Daniel Vrcic - 0.1.1-1%{?dist} +- ARGO-732 Structure the body of alarm message as JSON object +* Wed Mar 1 2017 Daniel Vrcic - 0.1.0-3%{?dist} +- timestamp is automatically generated and not taken from nagios +* Wed Mar 1 2017 Daniel Vrcic - 0.1.0-2%{?dist} +- added missing queue spools +- added missing spec dependancies +* Wed Feb 15 2017 Daniel Vrcic - 0.1.0-1%{?dist} +- first version From 4ee8d1b003e6736537d7c36a5d0ffab9bddf9c64 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Tue, 10 May 2022 13:50:11 +0200 Subject: [PATCH 29/32] have generic pathname for inspection socket --- pymod/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymod/stats.py b/pymod/stats.py index ee13d33..ff30d6e 100644 --- a/pymod/stats.py +++ b/pymod/stats.py @@ -12,7 +12,7 @@ from ams_publisher.shared import Shared MAXCMDLENGTH = 128 -STATSOCK = '/run/argo-nagios-ams-publisher/sock' +STATSOCK = '/run/ams-publisher/sock' def query_stats(last_minutes): From e87afc00ff5bee17280b560f1be37abd2816781c Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 13 May 2022 12:21:57 +0200 Subject: [PATCH 30/32] define requirements for each package explicitly --- ams-publisher.spec | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index 6ebbb27..c0097c9 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -16,16 +16,6 @@ Source0: %{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch -BuildRequires: python3-devel -Requires: python3-argo-ams-library -Requires: python3-avro -Requires: python3-dirq -Requires: python3-messaging -Requires: python36-pytz - -Requires(post): systemd -Requires(preun): systemd -Requires(postun): systemd %prep %setup -q @@ -46,8 +36,18 @@ install --directory --mode 755 $RPM_BUILD_ROOT/%{_localstatedir}/spool/ams-publi %package -n argo-nagios-ams-publisher Summary: Bridge from Nagios to the ARGO Messaging system -Conflicts: argo-sensu-ams-publisher -Requires: nagios +Conflicts: argo-sensu-ams-publisher + +BuildRequires: python3-devel +Requires: nagios +Requires: python3-argo-ams-library +Requires: python3-avro +Requires: python3-dirq +Requires: python3-messaging +Requires: python36-pytz +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd %description -n argo-nagios-ams-publisher Bridge from Nagios to the ARGO Messaging system @@ -78,7 +78,17 @@ Bridge from Nagios to the ARGO Messaging system %package -n argo-sensu-ams-publisher Summary: Bridge from Sensu to the ARGO Messaging system Conflicts: argo-nagios-ams-publisher -Requires: sensu-go-backend + +BuildRequires: python3-devel +Requires: sensu-go-backend +Requires: python3-argo-ams-library +Requires: python3-avro +Requires: python3-dirq +Requires: python3-messaging +Requires: python36-pytz +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd %description -n argo-sensu-ams-publisher Bridge from Sensu to the ARGO Messaging system From 4266b3737ea901c1615f5fdefaca748708cb2a90 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 1 Sep 2022 07:14:48 +0200 Subject: [PATCH 31/32] list all changes of this release --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d339b1..423baab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.4.0] - 2022-09-1 + +### Changed + +* ARGO-3754 Build two RPMS, Nagios and Sensu with appropriate runtime permission settings +* ARGO-3825 List requires explicitly for each ams-publisher package + ## [0.3.9] - 2021-02-01 ### Changed @@ -11,7 +18,7 @@ ### Fixed -* remove leftovers from erroneous SIGHUP handling +* remove leftovers from erroneous SIGHUP handling ## [0.3.7] - 2020-07-08 @@ -135,7 +142,7 @@ ## [0.1.1] - 2017-03-14 -### Added +### Added * ARGO-732 Structure the body of alarm message as JSON object From c6100387b802021a5434a9e91d8685df3cb8a708 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 1 Sep 2022 07:15:35 +0200 Subject: [PATCH 32/32] version bump --- ams-publisher.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ams-publisher.spec b/ams-publisher.spec index c0097c9..eb5ed51 100644 --- a/ams-publisher.spec +++ b/ams-publisher.spec @@ -6,7 +6,7 @@ Name: ams-publisher Summary: Bridge from Sensu/Nagios to the ARGO Messaging system -Version: 0.3.9 +Version: 0.4.0 Release: 1%{mydist} Group: Network/Monitoring @@ -120,6 +120,9 @@ Bridge from Sensu to the ARGO Messaging system rm -rf $RPM_BUILD_ROOT %changelog +* Thu Sep 1 2022 Daniel Vrcic - 0.4.0-1%{?dist} +- ARGO-3754 Build two RPMS, Nagios and Sensu with appropriate runtime permission settings +- ARGO-3825 List requires explicitly for each ams-publisher package * Mon Feb 1 2021 Daniel Vrcic - 0.3.9-1%{?dist} - ARGO-2855 ams-publisher py3 switch - ARGO-2929 Let systemd handle runtime directory