From 1e0267b57cd9493ab1aa1ef765d90f93925bc5ce Mon Sep 17 00:00:00 2001 From: Robert Frank Date: Mon, 10 Nov 2014 14:11:36 +0000 Subject: [PATCH 1/2] major rewrite for perfsonar 3.4 no backward compatibility with older perfsonar versions --- manifests/apache.pp | 27 ++++++- manifests/config.pp | 88 +++++++---------------- manifests/install.pp | 6 +- manifests/params.pp | 127 +++++++++++++++++++++++--------- manifests/service.pp | 167 ++++++++++++++++++++++++------------------- 5 files changed, 242 insertions(+), 173 deletions(-) diff --git a/manifests/apache.pp b/manifests/apache.pp index 1b7cf89..c014cdc 100644 --- a/manifests/apache.pp +++ b/manifests/apache.pp @@ -6,6 +6,17 @@ $verifydepth = $perfsonar::params::verifydepth, $authdn = [], ) inherits perfsonar::params { + + file { "${perfsonar::params::conf_dir}/tk_redirect.conf": + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0644', + content => "RedirectMatch 301 ^/$ http://${::fqdn}/toolkit/\n", + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], + } + augeas { 'set mod_ssl params': incl => "${perfsonar::params::mod_dir}/ssl.conf", lens => 'Httpd.lns', @@ -20,6 +31,8 @@ "set directive[.='SSLVerifyDepth'] 'SSLVerifyDepth'", "set *[.='SSLVerifyDepth']/arg ${verifydepth}", ], + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], } $have_auth = $authdn ? { undef => 0, @@ -46,14 +59,16 @@ # instead of the set commands above, the setm regex versions below should work as well (they do in augtool), # but for some reason they produce an error when run by puppet ('Could not evaluate: missing string argument 2 for setm', no useful debug output either) # the rm commands below work, but we shouldn't use them with the single set commands above because they can cause security problems - # e.g., if the original auth section is removed without from an unexpected directory entry without adding the include + # e.g., if the original auth section is removed from an unexpected directory entry without adding the include #"rm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')]/directive[.='AuthShadow']", #"rm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')]/directive[.='AuthType']", #"rm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')]/directive[.='AuthName']", #"rm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')]/directive[.='Require']", #"setm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')] directive[.='Include'] 'Include'", #"setm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')] *[.='Include']/arg '${perfsonar::params::httpd_dir}/ssl_auth.conf'", - ] + ], + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], } file { "${perfsonar::params::httpd_dir}/ssl_auth.conf": ensure => 'present', @@ -61,6 +76,8 @@ group => 'root', mode => '0644', content => template("${module_name}/ssl_auth.conf.erb"), + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], } } else { augeas { 'restore mod_ssl auth': @@ -102,10 +119,14 @@ #"setm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')] *[.='Require']/arg[1] 'group'", #"setm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')] *[.='Require']/arg[2] 'wheel'", #"setm Directory[arg=~regexp('.*/web/root/admin(/.*)?\"?')] *[.='Require']/arg[3] 'admin'", - ] + ], + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], } file { "${perfsonar::params::httpd_dir}/ssl_auth.conf": ensure => 'absent', + notify => Service[$::perfsonar::params::httpd_service], + require => Package[$::perfsonar::params::httpd_package], } } } diff --git a/manifests/config.pp b/manifests/config.pp index 26f70ad..37ad9ef 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,42 +1,6 @@ class perfsonar::config( $admininfo = {}, - $agentconfig = {}, ) inherits perfsonar::params { - exec { 'ps_initdb_cacti': - environment => [ "HOME=/root" ], - command => $perfsonar::params::ps_initdb_cmd_cacti, - logoutput => 'on_failure', - require => Service['mysqld'], - unless => '/bin/echo "show databases" | /usr/bin/mysql | /bin/grep -q "^cacti$"', - } - exec { 'ps_initdb_psb_bwctl': - environment => [ "HOME=/root" ], - command => $perfsonar::params::ps_initdb_cmd_psb_bwctl, - logoutput => 'on_failure', - require => Service['mysqld'], - unless => '/bin/echo "show databases" | /usr/bin/mysql | /bin/grep -q "^bwctl$"', - } - exec { 'ps_initdb_psb_owamp': - environment => [ "HOME=/root" ], - command => $perfsonar::params::ps_initdb_cmd_psb_owamp, - logoutput => 'on_failure', - require => Service['mysqld'], - unless => '/bin/echo "show databases" | /usr/bin/mysql | /bin/grep -q "^owamp$"', - } - exec { 'ps_initdb_tr_ma': - environment => [ "HOME=/root" ], - command => $perfsonar::params::ps_initdb_cmd_tr_ma, - logoutput => 'on_failure', - require => Service['mysqld'], - unless => '/bin/echo "show databases" | /usr/bin/mysql | /bin/grep -q "^traceroute_ma$"', - } - exec { 'ps_initdb_pinger': - environment => [ "HOME=/root" ], - command => $perfsonar::params::ps_initdb_cmd_pinger, - logoutput => 'on_failure', - require => Service['mysqld'], - unless => '/bin/echo "show databases" | /usr/bin/mysql | /bin/grep -q "^pingerMA$"', - } # the perfsonar kickstart post script replaces all 'yes' values in /etc/sysconfig/readahead to 'no' # we do the same here with augeas, it replaces all '"yes"' and 'yes' values to 'no' # let's hope they don't start using single quotes in that file, @@ -50,36 +14,38 @@ # we need the onlyif because the above command produces an error if the regex can't find any values onlyif => 'match readahead/*[label()!=\'#comment\'][.=~regexp(\'"?yes"?\')] size > 0', } - # ensure ssh is enabled by default, otherwise saving the configuration in the webinterface (or boot can disable it) - augeas { 'enable ssh': - context => '/files/opt/perfsonar_ps/toolkit/etc/enabled_services', - incl => '/opt/perfsonar_ps/toolkit/etc/enabled_services', - lens => 'Shellvars.lns', - changes => 'set ssh_enabled "enabled"' - } - $site_project = 'pS-NPToolkit-3.3.1' file { '/opt/perfsonar_ps/toolkit/etc/administrative_info': ensure => 'present', owner => 'root', group => 'root', mode => '0644', content => template("${module_name}/administrative_info.erb"), + require => Package['perl-perfSONAR_PS-Toolkit'] + } + # update owner / permissions on directories + file { '/var/lib/perfsonar/db_backups': + ensure => 'directory', + owner => 'perfsonar', + group => 'perfsonar', + mode => '0755', + require => Package['perl-perfSONAR_PS-Toolkit'] + } + file { '/var/lib/perfsonar/log_view': + ensure => 'directory', + owner => 'perfsonar', + group => 'perfsonar', + mode => '0755', + require => Package['perl-perfSONAR_PS-Toolkit'] } -# file { '/opt/perfsonar_ps/toolkit/etc/external_addresses': -# ensure => 'present', -# owner => 'root', -# group => 'root', -# mode => '0644', -# content => template("${module_name}/administrative_info.erb"), -# } - $agent_options = merge($perfsonar::params::agentconfig, $agentconfig) - file { '/opt/perfsonar_ps/mesh_config/etc/agent_configuration.conf': - ensure => 'present', - owner => 'perfsonar', - group => 'perfsonar', - mode => '0644', - content => template("${module_name}/agent_configuration.conf.erb"), - } - # ??? run "sudo -u perfsonar /opt/perfsonar_ps/mesh_config/bin/generate_configuration" when agent_configuration.conf is changed - # it takes a long time to complete, so it's probably not a good idea, it's being run by a cron job every night any way } + +# info for 3.4 + +#run /opt/perfsonar_ps/toolkit/scripts/upgrade/upgrade_owamp_port_range.sh (new port range) + +#for script in %{install_base}/scripts/system_environment/*; do +# run script +#done + +# WLCG info: https://twiki.opensciencegrid.org/bin/view/Documentation/InstallUpdatePS +# only use BWCTL and OWAMP diff --git a/manifests/install.pp b/manifests/install.pp index bc38d4a..d4cb1b9 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -1,5 +1,7 @@ class perfsonar::install ( - $packages = $perfsonar::params::install_packages, + $ensure = $::perfsonar::install_ensure, ) inherits perfsonar::params { - package { $packages: } + package { $perfsonar::params::install_packages: + ensure => $ensure, + } } diff --git a/manifests/params.pp b/manifests/params.pp index a8caa27..6a39e5d 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,53 +1,108 @@ -class perfsonar::params { +class perfsonar::params( + $regular_testing_install_ensure = 'present', + $regular_testing_ensure = 'stopped', + $regular_testing_enable = false, + $regular_testing_loglvl = 'INFO', + $regular_testing_logger = 'Log::Dispatch::FileRotate', + $regular_testing_logfile = '/var/log/perfsonar/regular_testing.log', + $regular_testing_snotify = true, + $mesh_config_install_ensure = 'present', + $mesh_config_agent = {}, + $owamp_install_ensure = 'present', + $owamp_ensure = 'stopped', + $owamp_enable = false, + $bwctl_install_ensure = 'present', + $bwctl_ensure = 'stopped', + $bwctl_enable = false, + $esmond_dbname = 'esmond', + $esmond_dbuser = 'esmond', + $esmond_dbpass = 'jqIqSIiuzwI0FMUu', +) { # package list taken from centos6-netinstall.cfg (from the perfsonar netinstall cd) # system packages (already installed on standard installation) and # packages that are dependencies of packages in this list have been removed from the original list + # general perfsonar packages $install_packages = [ 'perl-perfSONAR_PS-Toolkit', - 'perl-perfSONAR_PS-Toolkit-SystemEnvironment', - 'perl-perfSONAR_PS-MeshConfig-Agent', - 'kmod-sk98lin', - 'iperf3', - 'gcc', - 'mysql-devel', - 'device-mapper-multipath', - 'nuttcp', - 'php-gd', - 'php-xml', - 'syslinux', - 'tcptrace', - 'xplot-tcptrace', + # installed as dependencies, but need them here to get the dependencies in puppet right + 'httpd', + 'esmond', +# don't want to install SystemEnvironment because it keeps overwriting my configurations during updates +# 'perl-perfSONAR_PS-Toolkit-SystemEnvironment', +# don't want to install gcc and mysql, it's not required +# 'gcc', +# 'mysql-devel', +# is this for the web100 kernel only ?? +# 'kmod-sk98lin', +# are the ones below still required ? +# 'device-mapper-multipath', +# 'php-gd', +# 'php-xml', +# 'syslinux', +# 'xplot-tcptrace', ] # other packages in the original kickstart, but left out # 'perl-DBD-mysql' doesn't exist, it's called perl-DBD-MySQL # 'xkeyboard-config' do we need it, we don't run X ?? # 'comps-extras' contains images only, do we need it ?? - # init database commands - # names of db init scripts to run can be found in /opt/perfsonar_ps/toolkit/scripts/initialize_databases - # we can just run two of them directly, the others call perl scripts which we have run directly - $ps_initdb_cmd_cacti = '/opt/perfsonar_ps/toolkit/scripts/initialize_cacti_database' - $ps_initdb_cmd_pinger = '/opt/perfsonar_ps/toolkit/scripts/initialize_pinger_database' - # the following perl commands ask for the mysql root password on stdin, so we extract it from /root/.my.cnf - # the original shell scripts (/opt/perfsonar_ps/toolkit/scripts/initialize_* scripts just pipe echo into the - # perl script and would require a mysql server without a root password - $ps_initdb_cmd_psb_bwctl = '/bin/sed -n "s/^password=//p" /root/.my.cnf | tr -d "\n\'" | /opt/perfsonar_ps/perfsonarbuoy_ma/bin/bwdb.pl -i root' - $ps_initdb_cmd_psb_owamp = '/bin/sed -n "s/^password=//p" /root/.my.cnf | tr -d "\n\'" | /opt/perfsonar_ps/perfsonarbuoy_ma/bin/owdb.pl -i root' - $ps_initdb_cmd_tr_ma = '/bin/sed -n "s/^password=//p" /root/.my.cnf | tr -d "\n\'" | /opt/perfsonar_ps/traceroute_ma/bin/tracedb.pl -i root -c /opt/perfsonar_ps/perfsonarbuoy_ma/etc' + $regular_testing_packages = [ + 'perl-perfSONAR_PS-RegularTesting', + 'perl-DBD-MySQL', # required by regular testing ? I've seen related error message in the logs when it's not installed + ] + $mesh_config_packages = [ + 'perl-perfSONAR_PS-MeshConfig-Agent', + ] + # we should split client and server at some point + $owamp_packages = [ + 'owamp-client', + 'owamp-server', + 'owamp', # this installs both, the client and the server, plus I2util (which is installed by neither the client nor the server) + ] + # we should split client and server at some point + $bwctl_packages = [ + 'bwctl-client', + 'bwctl-server', + 'bwctl', # this installs both, the client and the server + 'iperf3', # bwctl packages install iperf and iperf3-devel as dependency, but not iperf3 ??? + ] - # apache options + # apache default options $hostcert = '/etc/grid-security/hostcert.pem' $hostkey = '/etc/grid-security/hostkey.pem' $capath = '/etc/grid-security/certificates' $clientauth = 'optional' $verifydepth = '5' + # service status defaults + $config_daemon_ensure = 'running' + $config_daemon_enable = true + $config_nic_params = true + $generate_motd_enable = false + $htcacheclean_ensure = 'stopped' + $htcacheclean_enable = false + $httpd_ensure = 'running' + $httpd_enable = true + $ls_cache_daemon_ensure = 'running' + $ls_cache_daemon_enable = true + $ls_reg_daemon_ensure = 'running' + $ls_reg_daemon_enable = true + $multipathd_ensure = 'stopped' + $multipathd_enable = false + $ndt_ensure = 'stopped' + $ndt_enable = false + $npad_ensure = 'stopped' + $npad_enable = false + $nscd_ensure = 'stopped' + $nscd_enable = false + $ls_bs_client_ensure = 'stopped' + $ls_bs_client_enable = false + $cassandra_ensure = 'running' + $cassandra_enable = true + # default mesh config $agentconfig = { mesh => [], - traceroute_master_conf => '/opt/perfsonar_ps/traceroute_ma/etc/traceroute-master.conf', - owmesh_conf => '/opt/perfsonar_ps/perfsonarbuoy_ma/etc/owmesh.conf', - pinger_landmarks => '/opt/perfsonar_ps/PingER/etc/pinger-landmarks.xml', restart_services => 0, use_toolkit => 1, send_error_emails => 1, @@ -56,10 +111,16 @@ # paths case $::osfamily { 'RedHat': { - $httpd_dir = '/etc/httpd' - $mod_dir = "${httpd_dir}/conf.d" - $conf_dir = "${httpd_dir}/conf.d" + $httpd_package = 'httpd' + $httpd_service = 'httpd' + $httpd_hasrestart = true + $httpd_hasstatus = true + $httpd_dir = '/etc/httpd' + $mod_dir = "${httpd_dir}/conf.d" + $conf_dir = "${httpd_dir}/conf.d" + } + default: { + fail("osfamily ${::osfamily} is not supported") } - default: {} } } diff --git a/manifests/service.pp b/manifests/service.pp index 78d390c..11d4adb 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -1,114 +1,133 @@ class perfsonar::service( -) { + $config_daemon_ensure = $::perfsonar::params::config_daemon_ensure, + $config_daemon_enable = $::perfsonar::params::config_daemon_enable, + $config_nic_params = $::perfsonar::params::config_nic_params, + $generate_motd_enable = $::perfsonar::params::generate_motd_enable, + $htcacheclean_ensure = $::perfsonar::params::htcacheclean_ensure, + $htcacheclean_enable = $::perfsonar::params::htcacheclean_enable, + $httpd_ensure = $::perfsonar::params::httpd_ensure, + $httpd_enable = $::perfsonar::params::httpd_enable, + $ls_cache_daemon_ensure = $::perfsonar::params::ls_cache_daemon_ensure, + $ls_cache_daemon_enable = $::perfsonar::params::ls_cache_daemon_enable, + $ls_reg_daemon_ensure = $::perfsonar::params::ls_reg_daemon_ensure, + $ls_reg_daemon_enable = $::perfsonar::params::ls_reg_daemon_enable, + $multipathd_ensure = $::perfsonar::params::multipathd_ensure, + $multipathd_enable = $::perfsonar::params::multipathd_enable, + $ndt_ensure = $::perfsonar::params::ndt_ensure, + $ndt_enable = $::perfsonar::params::ndt_enable, + $npad_ensure = $::perfsonar::params::npad_ensure, + $npad_enable = $::perfsonar::params::npad_enable, + $nscd_ensure = $::perfsonar::params::nscd_ensure, + $nscd_enable = $::perfsonar::params::nscd_enable, + $ls_bs_client_ensure = $::perfsonar::params::ls_bs_client_ensure, + $ls_bs_client_enable = $::perfsonar::params::ls_bs_client_enable, + $cassandra_ensure = $::perfsonar::params::cassandra_ensure, + $cassandra_enable = $::perfsonar::params::cassandra_enable, +) inherits perfsonar::params { # start stop restart - service { 'bwctld': - ensure => 'running', - enable => true, + service { 'config_daemon': + ensure => $config_daemon_ensure, + enable => $config_daemon_enable, hasstatus => false, hasrestart => true, } - # start stop restart - service { 'config_daemon': - } - # start + # start (no service, only runs at boot) service { 'configure_nic_parameters': + enable => $config_nic_params, + hasstatus => false, + hasrestart => false, } # start stop(nil) restart(start) - service { 'dicover_external_address': - } - # start stop(nil) restart +# not present in 3.4 +# service { 'dicover_external_address': +# } + # start stop(nil) restart (no service, only runs at boot) service { 'generate_motd': + enable => $generate_motd_enable, + hasstatus => false, + hasrestart => true, } # start stop status restart condrestart|try-restart(stop start) force-reload|reload(nil) service { 'htcacheclean': + ensure => $htcacheclean_ensure, + enable => $htcacheclean_enable, + hasstatus => true, + hasrestart => true, } - service { 'httpd': + service { $::perfsonar::params::httpd_service: + ensure => $httpd_ensure, + enable => $httpd_enable, + hasstatus => $::perfsonar::params::httpd_hasstatus, + hasrestart => $::perfsonar::params::httpd_hasrestart, + require => Package[$::perfsonar::params::httpd_package], } # start stop restart service { 'ls_cache_daemon': + ensure => $ls_cache_daemon_ensure, + enable => $ls_cache_daemon_enable, + hasstatus => false, + hasrestart => true, } # start stop restart service { 'ls_registration_daemon': + ensure => $ls_reg_daemon_ensure, + enable => $ls_reg_daemon_enable, + hasstatus => false, + hasrestart => true, } + # do we need it ??? # start stop status restart condrestart|try-restart(restart) force-reload|reload service { 'multipathd': + ensure => $multipathd_ensure, + enable => $multipathd_enable, + hasstatus => true, + hasrestart => true, } # start stop status restart|reload service { 'ndt': + ensure => $ndt_ensure, + enable => $ndt_enable, + hasstatus => true, + hasrestart => true, } +# doesn't seem to be used any more +# file { '/opt/perfsonar_ps/toolkit/etc/enabled_services': +# ensure => 'present', +# owner => 'perfsonar', +# group => 'perfsonar', +# mode => '0644', +# content => template("${module_name}/enabled_services.erb"), +# } # start stop restart service { 'npad': + ensure => $npad_ensure, + enable => $npad_enable, + hasstatus => false, + hasrestart => true, } # start stop status restart condrestart|try-restart(restart) force-reload|reload service { 'nscd': + ensure => $nscd_ensure, + enable => $nscd_enable, + hasstatus => true, + hasrestart => true, } - # start stop status restart condrestart|try-restart(restart) force-reload(restart) reload(restart) - service { 'openct': - } - # start stop restart - service { 'owamp': - } - # start stop restart status condrestart|try-restart - service { 'pcscd': - } - # start stop restart - service { 'perfsonarbuoy_bw_collector': - } - # start stop restart - service { 'perfsonarbuoy_bw_master': - } - # start stop restart - service { 'perfsonarbuoy_ma': - } - # start stop restart - service { 'perfsonarbuoy_owp_collector': - } - # start stop restart configure - service { 'perfsonarbuoy_owp_master': - } - # start stop restart - service { 'PingER': - } - # start stop restart condrestart|try-restart(restart) reload(nil) force-reload(restart) status - service { 'portreserve': - } + # do we need it ??? # start stop status restart|reload|force-reload condrestart|try-restart service { 'rpcbind': } # start stop restart - service { 'services_init_script': - } - # start stop restart service { 'simple_ls_bootstrap_client': + ensure => $ls_bs_client_ensure, + enable => $ls_bs_client_enable, + hasstatus => false, + hasrestart => true, + pattern => 'SimpleLSBootStrapClientDaemon.pl', } - # start stop restart - service { 'snmp_ma': - } - # start stop restart - service { 'topology_service': - } - # start stop restart - service { 'traceroute_ma': - } - # start stop restart - service { 'traceroute_master': - } - # start stop restart - service { 'traceroute_ondemand_mp': - } - # start stop restart - service { 'tracerouet_scheduler': + service { 'cassandra': + ensure => $cassandra_ensure, + enable => $cassandra_enable, + hasstatus => true, + hasrestart => true, } - - # the following services are installed by perfsonar, but not enabled - # avahi daemon - # bluetooth - # cups - # nfs - # nfslock - # rpcgssd (nfs gss) - # rpcidmapd - # rpcsvcgssd - # snmpd - # snmptrapd } From 411bb4cdb23d2e6c0b7360cf1e56e1412a1e490a Mon Sep 17 00:00:00 2001 From: Robert Frank Date: Mon, 10 Nov 2014 14:16:45 +0000 Subject: [PATCH 2/2] added missing file the previous git commit -a didn't add new files --- manifests/bwctl.pp | 4 ++ manifests/bwctl/install.pp | 7 +++ manifests/bwctl/service.pp | 11 +++++ manifests/esmond.pp | 53 +++++++++++++++++++++++ manifests/init.pp | 11 +++++ manifests/mesh_config.pp | 4 ++ manifests/mesh_config/config.pp | 32 ++++++++++++++ manifests/mesh_config/install.pp | 7 +++ manifests/owamp.pp | 4 ++ manifests/owamp/install.pp | 7 +++ manifests/owamp/service.pp | 11 +++++ manifests/regular_testing.pp | 5 +++ manifests/regular_testing/config.pp | 35 +++++++++++++++ manifests/regular_testing/install.pp | 7 +++ manifests/regular_testing/service.pp | 13 ++++++ templates/configure_esmond.erb | 42 ++++++++++++++++++ templates/configure_regular_testing.erb | 38 ++++++++++++++++ templates/enabled_services.erb | 14 ++++++ templates/esmond.conf.erb | 44 +++++++++++++++++++ templates/regular_testing-logger.conf.erb | 22 ++++++++++ 20 files changed, 371 insertions(+) create mode 100644 manifests/bwctl.pp create mode 100644 manifests/bwctl/install.pp create mode 100644 manifests/bwctl/service.pp create mode 100644 manifests/esmond.pp create mode 100644 manifests/init.pp create mode 100644 manifests/mesh_config.pp create mode 100644 manifests/mesh_config/config.pp create mode 100644 manifests/mesh_config/install.pp create mode 100644 manifests/owamp.pp create mode 100644 manifests/owamp/install.pp create mode 100644 manifests/owamp/service.pp create mode 100644 manifests/regular_testing.pp create mode 100644 manifests/regular_testing/config.pp create mode 100644 manifests/regular_testing/install.pp create mode 100644 manifests/regular_testing/service.pp create mode 100644 templates/configure_esmond.erb create mode 100644 templates/configure_regular_testing.erb create mode 100644 templates/enabled_services.erb create mode 100644 templates/esmond.conf.erb create mode 100644 templates/regular_testing-logger.conf.erb diff --git a/manifests/bwctl.pp b/manifests/bwctl.pp new file mode 100644 index 0000000..25dd3b7 --- /dev/null +++ b/manifests/bwctl.pp @@ -0,0 +1,4 @@ +class perfsonar::bwctl { + include 'perfsonar::bwctl::install' + include 'perfsonar::bwctl::service' +} diff --git a/manifests/bwctl/install.pp b/manifests/bwctl/install.pp new file mode 100644 index 0000000..2adbc0a --- /dev/null +++ b/manifests/bwctl/install.pp @@ -0,0 +1,7 @@ +class perfsonar::bwctl::install( + $ensure = $::perfsonar::params::bwctl_install_ensure, +) inherits perfsonar::params { + package { $::perfsonar::params::bwctl_packages: + ensure => $ensure, + } +} diff --git a/manifests/bwctl/service.pp b/manifests/bwctl/service.pp new file mode 100644 index 0000000..fbc5f76 --- /dev/null +++ b/manifests/bwctl/service.pp @@ -0,0 +1,11 @@ +class perfsonar::bwctl::service( + $ensure = $::perfsonar::params::bwctl_ensure, + $enable = $::perfsonar::params::bwctl_enable, +) inherits perfsonar::params { + service { 'bwctld': + ensure => $ensure, + enable => $enable, + hasstatus => false, + hasrestart => true, + } +} diff --git a/manifests/esmond.pp b/manifests/esmond.pp new file mode 100644 index 0000000..42180ae --- /dev/null +++ b/manifests/esmond.pp @@ -0,0 +1,53 @@ +class perfsonar::esmond ( + $use_db_module = true, + $dbname = $::perfsonar::params::esmond_dbname, + $dbuser = $::perfsonar::params::esmond_dbuser, + $dbpassword = $::perfsonar::params::esmond_dbpass, +) inherits perfsonar::params { + if $use_db_module { + class { 'postgresql::server': } + postgresql::server::db { $dbname: + user => $dbuser, + password => postgresql_password($dbuser, $dbpassword), + grant => 'ALL', + before => Exec['run esmond configuration script'], + } + # update auth to allow esmond access to the DB + postgresql::server::pg_hba_rule { 'allow local password auth': + description => 'allow local authentication using a password', + type => 'local', + database => 'all', + user => 'all', + auth_method => 'md5', + # need local md5 auth for esmond user, but the second default pg_hba rule + # is a generic ident auth for local connections, therefore we need to place + # this rule before the second default rule + order => '002', + before => Exec['run esmond configuration script'], + } + } + + file { '/opt/esmond/esmond.conf': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0644', + content => template("${module_name}/esmond.conf.erb"), + require => Package['esmond'], + } + # the remaining content of this script should be moved here if possible + file { '/usr/local/sbin/puppet_perfsonar_configure_esmond': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0750', + content => template("${module_name}/configure_esmond.erb"), + require => File['/opt/esmond/esmond.conf'], + } + exec { 'run esmond configuration script': + command => '/usr/local/sbin/puppet_perfsonar_configure_esmond', + logoutput => 'on_failure', + creates => '/var/lib/esmond/.configured.puppet', + require => File['/usr/local/sbin/puppet_perfsonar_configure_esmond'], + } +} diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 0000000..f7535ef --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,11 @@ +class perfsonar { + include 'perfsonar::install' + include 'perfsonar::config' + include 'perfsonar::service' + include 'perfsonar::apache' + include 'perfsonar::esmond' + include 'perfsonar::regular_testing' + include 'perfsonar::mesh_config' + include 'perfsonar::owamp' + include 'perfsonar::bwctl' +} diff --git a/manifests/mesh_config.pp b/manifests/mesh_config.pp new file mode 100644 index 0000000..19eba29 --- /dev/null +++ b/manifests/mesh_config.pp @@ -0,0 +1,4 @@ +class perfsonar::mesh_config { + include 'perfsonar::mesh_config::install' + include 'perfsonar::mesh_config::config' +} diff --git a/manifests/mesh_config/config.pp b/manifests/mesh_config/config.pp new file mode 100644 index 0000000..51fcb29 --- /dev/null +++ b/manifests/mesh_config/config.pp @@ -0,0 +1,32 @@ +class perfsonar::mesh_config::config( + $agentconfig = $::perfsonar::params::mesh_config_agent, +) inherits perfsonar::params { + $agent_options = merge($perfsonar::params::agentconfig, $agentconfig) + file { '/opt/perfsonar_ps/mesh_config/etc/agent_configuration.conf': + ensure => 'present', + owner => 'perfsonar', + group => 'perfsonar', + mode => '0644', + content => template("${module_name}/agent_configuration.conf.erb"), + require => Package['perl-perfSONAR_PS-MeshConfig-Agent'] + } + # needs notty in sudoers + exec { 'generate mesh configuration': + command => '/usr/bin/sudo -u perfsonar /opt/perfsonar_ps/mesh_config/bin/generate_configuration', + logoutput => 'on_failure', + subscribe => File['/opt/perfsonar_ps/mesh_config/etc/agent_configuration.conf'], + require => [ + Exec['run regular testing configuration script'], + File['/etc/sudoers.d/perfsonar'], + ], + refreshonly => true, + notify => Service['regular_testing'], + } + file { '/etc/sudoers.d/perfsonar': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0440', + content => "Defaults!/opt/perfsonar_ps/mesh_config/bin/generate_configuration !requiretty\n", + } +} diff --git a/manifests/mesh_config/install.pp b/manifests/mesh_config/install.pp new file mode 100644 index 0000000..a27557b --- /dev/null +++ b/manifests/mesh_config/install.pp @@ -0,0 +1,7 @@ +class perfsonar::mesh_config::install( + $ensure = $::perfsonar::params::mesh_config_install_ensure, +) inherits perfsonar::params { + package { $::perfsonar::params::mesh_config_packages: + ensure => $ensure, + } +} diff --git a/manifests/owamp.pp b/manifests/owamp.pp new file mode 100644 index 0000000..de1ebd1 --- /dev/null +++ b/manifests/owamp.pp @@ -0,0 +1,4 @@ +class perfsonar::owamp { + include 'perfsonar::owamp::install' + include 'perfsonar::owamp::service' +} diff --git a/manifests/owamp/install.pp b/manifests/owamp/install.pp new file mode 100644 index 0000000..7165351 --- /dev/null +++ b/manifests/owamp/install.pp @@ -0,0 +1,7 @@ +class perfsonar::owamp::install( + $ensure = $::perfsonar::params::owamp_install_ensure, +) inherits perfsonar::params { + package { $::perfsonar::params::owamp_packages: + ensure => $ensure, + } +} diff --git a/manifests/owamp/service.pp b/manifests/owamp/service.pp new file mode 100644 index 0000000..b512c04 --- /dev/null +++ b/manifests/owamp/service.pp @@ -0,0 +1,11 @@ +class perfsonar::owamp::service( + $ensure = $::perfsonar::params::owamp_ensure, + $enable = $::perfsonar::params::owamp_enable, +) inherits perfsonar::params { + service { 'owampd': + ensure => $ensure, + enable => $enable, + hasstatus => false, + hasrestart => true, + } +} diff --git a/manifests/regular_testing.pp b/manifests/regular_testing.pp new file mode 100644 index 0000000..1df3d67 --- /dev/null +++ b/manifests/regular_testing.pp @@ -0,0 +1,5 @@ +class perfsonar::regular_testing { + include 'perfsonar::regular_testing::install' + include 'perfsonar::regular_testing::config' + include 'perfsonar::regular_testing::service' +} diff --git a/manifests/regular_testing/config.pp b/manifests/regular_testing/config.pp new file mode 100644 index 0000000..28d7f8a --- /dev/null +++ b/manifests/regular_testing/config.pp @@ -0,0 +1,35 @@ +# loglevel is a puppet metaparameter, so have to use something else (loglvl) +class perfsonar::regular_testing::config( + $snotify = $::perfsonar::params::regular_testing_snotify, + $loglvl = $::perfsonar::params::regular_testing_loglvl, + $logger = $::perfsonar::params::regular_testing_logger, + $logfile = $::perfsonar::params::regular_testing_logfile, +) inherits perfsonar::params { + file { '/usr/local/sbin/puppet_perfsonar_configure_regular_testing': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0750', + content => template("${module_name}/configure_regular_testing.erb"), + require => Package['perl-perfSONAR_PS-RegularTesting'] + } + exec { 'run regular testing configuration script': + command => '/usr/local/sbin/puppet_perfsonar_configure_regular_testing', + logoutput => 'on_failure', + creates => '/var/lib/perfsonar/regular_testing/.configured.puppet', + require => File['/usr/local/sbin/puppet_perfsonar_configure_regular_testing'], + } + $tn = $snotify ? { + false => undef, + default => Service['regular_testing'], + } + file { '/opt/perfsonar_ps/regular_testing/etc/regular_testing-logger.conf': + ensure => 'file', + owner => 'perfsonar', + group => 'perfsonar', + mode => '0644', + content => template("${module_name}/regular_testing-logger.conf.erb"), + require => Exec['run regular testing configuration script'], + notify => $tn, + } +} diff --git a/manifests/regular_testing/install.pp b/manifests/regular_testing/install.pp new file mode 100644 index 0000000..ebb07a3 --- /dev/null +++ b/manifests/regular_testing/install.pp @@ -0,0 +1,7 @@ +class perfsonar::regular_testing::install( + $ensure = $::perfsonar::params::regular_testing_install_ensure, +) inherits perfsonar::params { + package { $::perfsonar::params::regular_testing_packages: + ensure => $ensure, + } +} diff --git a/manifests/regular_testing/service.pp b/manifests/regular_testing/service.pp new file mode 100644 index 0000000..1cdd29c --- /dev/null +++ b/manifests/regular_testing/service.pp @@ -0,0 +1,13 @@ +class perfsonar::regular_testing::service( + $ensure = $::perfsonar::params::regular_testing_ensure, + $enable = $::perfsonar::params::regular_testing_enable, +) inherits perfsonar::params { + service { 'regular_testing': + ensure => $ensure, + enable => $enable, + hasstatus => false, + hasrestart => true, + pattern => 'perfSONAR_PS Regular Testing', + require => Exec['run regular testing configuration script'], + } +} diff --git a/templates/configure_esmond.erb b/templates/configure_esmond.erb new file mode 100644 index 0000000..0c91657 --- /dev/null +++ b/templates/configure_esmond.erb @@ -0,0 +1,42 @@ +#!/bin/bash + +<% if not @use_db_module then -%> +#init postgres +if [ -z "$(ls -A /var/lib/pgsql/data)" ]; then + /sbin/service postgresql initdb + /sbin/service postgresql start + echo "Waiting for postgresql to complete startup" + sleep 20 + sudo -u postgres psql -c "CREATE USER <%= @dbuser %> WITH PASSWORD '<%= @dbpassword %>'" + sudo -u postgres psql -c "CREATE DATABASE <%= @dbname %>" + sudo -u postgres psql -c "GRANT ALL ON DATABASE <%= @dbname %> to <%= @dbuser %>" + cp -f /opt/perfsonar_ps/toolkit/etc/default_service_configs/pg_hba.conf /var/lib/pgsql/data/pg_hba.conf + /sbin/service postgresql restart +# sed -i "s/sql_db_name = .*/sql_db_name = esmond/g" /opt/esmond/esmond.conf +# sed -i "s/sql_db_user = .*/sql_db_user = esmond/g" /opt/esmond/esmond.conf +# sed -i "s/sql_db_password = .*/sql_db_password = 7hc4m1/g" /opt/esmond/esmond.conf +fi + +<% end -%> +#disable JMX in cassandra so will start even if /etc/sysconfig/network HOSTNAME does not resolve +sed -i '/^JVM_OPTS="\$JVM_OPTS -Dcom.sun.management.jmx/ s/^/#/' /etc/cassandra/conf/cassandra-env.sh + +#set esmond env variables +export ESMOND_ROOT=/opt/esmond +export ESMOND_CONF=$ESMOND_ROOT/esmond.conf +export DJANGO_SETTINGS_MODULE=esmond.settings + +#initialize python +cd /opt/esmond +source /opt/rh/python27/enable +/opt/rh/python27/root/usr/bin/virtualenv --prompt="(esmond)" . +. bin/activate + +#build esmond tables +python esmond/manage.py syncdb --noinput + +#create api key +KEY=`python esmond/manage.py add_ps_metadata_post_user perfsonar | grep "Key:" | cut -f2 -d " "` +python esmond/manage.py add_timeseries_post_user perfsonar + +touch /var/lib/esmond/.configured.puppet diff --git a/templates/configure_regular_testing.erb b/templates/configure_regular_testing.erb new file mode 100644 index 0000000..48befa6 --- /dev/null +++ b/templates/configure_regular_testing.erb @@ -0,0 +1,38 @@ +#!/bin/bash + +#set esmond env variables +export ESMOND_ROOT=/opt/esmond +export ESMOND_CONF=$ESMOND_ROOT/esmond.conf +export DJANGO_SETTINGS_MODULE=esmond.settings + +#initialize python +cd /opt/esmond +source /opt/rh/python27/enable +/opt/rh/python27/root/usr/bin/virtualenv --prompt="(esmond)" . +. bin/activate + +#build esmond tables +python esmond/manage.py syncdb --noinput + +#create api key +KEY=`python esmond/manage.py add_ps_metadata_post_user perfsonar | grep "Key:" | cut -f2 -d " "` +python esmond/manage.py add_timeseries_post_user perfsonar + +#put api key in regular_testing +if [ -n "$KEY" ]; then + grep -q 'esmond/latency' /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf + if [ $? != 0 ]; then + mv /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf.install.back + cp -f /opt/perfsonar_ps/toolkit/etc/default_service_configs/regular_testing.conf /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf + fi + + grep -q ESMOND_API_KEY /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf + if [ $? == 0 ]; then + sed -i "s/ESMOND_API_KEY/$KEY/g" /opt/perfsonar_ps/regular_testing/etc/regular_testing.conf + fi +fi + +if [ -f /opt/perfsonar_ps/PingER/etc/pinger-landmarks.xml -o /opt/perfsonar_ps/perfsonarbuoy_ma/etc/owmesh.conf ]; then + /opt/perfsonar_ps/toolkit/scripts/upgrade/upgrade_regular_tests +fi +touch /var/lib/perfsonar/regular_testing/.configured.puppet diff --git a/templates/enabled_services.erb b/templates/enabled_services.erb new file mode 100644 index 0000000..a888f54 --- /dev/null +++ b/templates/enabled_services.erb @@ -0,0 +1,14 @@ +snmpma_enabled=enabled +ndt_enabled=<%= @ndt_enabled ? 'enabled' : 'disabled' %> +owamp_enabled=<%= @owamp_enabled ? 'enabled' : 'disabled' %> +npad_enabled=<%= @npad_enabled ? 'enabled' : 'disabled' %> +pinger_enabled=enabled +psb_enabled=enabled +https_enabled=enabled +bwctl_enabled=<%= @bwctl_enabled ? 'enabled' : 'disabled' %> +ssh_enabled=system +hls_enabled=enabled +traceroute_ma=enabled +traceroute_scheduler=enabled +regular_testing_enabled=enabled +yum_cron_enabled=enabled diff --git a/templates/esmond.conf.erb b/templates/esmond.conf.erb new file mode 100644 index 0000000..6a11b51 --- /dev/null +++ b/templates/esmond.conf.erb @@ -0,0 +1,44 @@ +[main] +sql_db_engine = django.db.backends.postgresql_psycopg2 +sql_db_name = <%= @dbname %> +sql_db_user = <%= @dbuser %> +sql_db_password = <%= @dbpassword %> +tsdb_root = %(ESMOND_ROOT)s/tsdb-data +tsdb_chunk_prefixes = %(ESMOND_ROOT)s/tsdb-data +api_anon_limit = 30 +api_throttle_at = +api_throttle_timeframe = +api_throttle_expiration = +cassandra_servers = localhost:9160 +cassandra_user = +cassandra_pass = +db_profile_on_testing = no +profile_persister = no +mib_dirs = %(ESMOND_ROOT)s/mibs +mibs = +syslog_facility = local7 +syslog_priority = debug +traceback_dir = %(ESMOND_ROOT)s/crashlog +pid_dir = %(ESMOND_ROOT)s/var/ +espersistd_uri = 127.0.0.1:11211 +espoll_persist_uri = MemcachedPersistHandler:127.0.0.1:11211 +htpasswd_file = %(ESMOND_ROOT)s/htpasswd +[persist_map] +FastPollHC = cassandra +FastPoll = cassandra +JnxFirewall = cassandra +JnxCOS = cassandra +Errors = cassandra +ALUFastPollHC = cassandra +ALUErrors = cassandra +ALUIfRefPoll = aluifref +SentryPoll = cassandra +ALUSAPRefPoll = alusapref +ALUSAPPoll = cassandra +IfRefPoll = ifref +[persist_queues] +cassandra = CassandraPollPersister:9 +ifref = IfRefPollPersister:1 +infifref = InfIfRefPollPersister:1 +aluifref = ALUIfRefPollPersister:1 +alusapref = ALUSAPRefPersister:1 diff --git a/templates/regular_testing-logger.conf.erb b/templates/regular_testing-logger.conf.erb new file mode 100644 index 0000000..98da332 --- /dev/null +++ b/templates/regular_testing-logger.conf.erb @@ -0,0 +1,22 @@ +log4perl.logger.perfSONAR_PS=<%= @loglvl %>, A1 + +# uncomment to display log messages on the screen +#log4perl.appender.A1=Log::Dispatch::Screen + +# uncomment to log via syslog +#log4perl.appender.A1=Log::Dispatch::Syslog +#log4perl.appender.A1.facility=local5 + +# comment to prevent logging to a file +#log4perl.appender.A1=Log::Dispatch::FileRotate +log4perl.appender.A1=<%= @logger %> + +# alter location of the log file below +#log4perl.appender.A1.filename=/var/log/perfsonar/regular_testing.log +log4perl.appender.A1.filename=<%= @logfile %> +log4perl.appender.A1.max=7 +log4perl.appender.A1.DatePattern=yyyy-MM-dd +log4perl.appender.A1.mode=append +log4perl.appender.A1.permissions=sub{ 0644; } +log4perl.appender.A1.layout=Log::Log4perl::Layout::PatternLayout +log4perl.appender.A1.layout.ConversionPattern=%d (%P) %p> %F{1}:%L %M - %m%n