Db external to cluster

clusters/common/Makefile

@@ -446,7 +446,11 @@ vpath % $(PROVISIONED_D)
 #  ie. all PROVISIONED, all POWEROFF, all UNPROVISION
 
 up provision: $(VAGRANTFILE) $(TARGET_PROVIDER_FILES) verifylocalenv $(PROVISIONED_FLAGS) ingest-state
-	echo started: $(HUMAN_TSTAMP)
+	# XXX pull from cluster definition & generator
+	#sudo mariadb -r -B -s -e "UPDATE cluster_table SET control_host = '192.168.56.64' WHERE NAME= 'vc';" slurm_acct_db
+	#sudo mariadb -r -B -s -e "UPDATE cluster_table SET control_host = '192.168.56.50' WHERE NAME= 'vx';" slurm_acct_db
+	#sudo mariadb -r -B -s -e 'select * from cluster_table;' slurm_acct_db
+	#echo started: $(HUMAN_TSTAMP)
 
 up! provision!: unprovision up ingest-state
 
@@ -635,4 +639,4 @@ $(XFR_VBOXADD_ISO): $(TARBALLS_VBOXADD_ISO)
 	    rsync -LHcau $< $@										; \
 	fi
 
-# vim: background=dark
+# vim: background=dark ts=2 sw=2 syntax=make

clusters/common/Vagrantfile.d/Vagrantfile.template

@@ -7,7 +7,7 @@
 require 'pathname'
 
 # following should allow for a full copy of the centos/{os,updates} repositories + pad
-REPODISK_SIZE = 20	# Gb
+REPODISK_SIZE = 32	# Gb
 
 clustername_fullpath_raw = %x[pwd]
 clustername_fullpath     = "#{clustername_fullpath_raw}"

clusters/common/bin/provision.sh

@@ -539,7 +539,7 @@ ConfigureCentOSRepos() {
       return
   fi
 
-  repos_size=$(du -x -s -m ${_have_repos} 2>&1 | awk '{print $1}')
+  repos_size=$(cd ${_have_repos}; du -x -s -m . 2>&1 | awk '{print $1}')
   if ! [[ ${repos_size} =~ ${numeric} ]] ; then
     ErrExit ${EX_CONFIG} "ingest repository is corrupt or empty: ${repos_size}"
   fi
@@ -2223,8 +2223,18 @@ SW() {
         ErrExit ${EX_SOFTWARE} "${what}/${_s}/${stop_flag} present"
       fi
       cmds=$(echo $(ls ${what}/${_s}))
+
+      if [ -L "${where}" ] ; then
+        local target=$(readlink -e "${where}")
+        if [ -f "${target}" ] ; then
+          Rc ErrExit ${EX_OSFILE} "rm -f ${where}"
+        elif [ -d "${target}" ] ; then
+          Rc ErrExit ${EX_OSFILE} "rmdir ${where}"
+        fi
+      fi
       Rc ErrExit ${EX_OSFILE} "mkdir -p ${where}"
       Rc ErrExit ${EX_OSFILE} "chmod 0755 ${where}"
+
       local c
       for c in ${cmds}
       do

clusters/common/env/site-vars.sh

@@ -3,3 +3,11 @@ export LANGUAGE=${LANGUAGE:-${LANG}}
 export LC_ALL=${LC_ALL:-${LANG}}
 
 export TZ=${TZ:-MST7MDT}
+
+export LANL_PROTO=http
+export LANL_PROXY_HOST=proxyout.lanl.gov
+export LANL_PROXY_PORT=8080
+export LANL_PROXY=${LANL_PROTO}://${LANL_PROXY_HOST}:${LANL_PROXY_PORT}
+export http_proxy=${LANL_PROXY}
+export https_proxy=${LANL_PROXY}
+export proxy=${LANL_PROXY}

clusters/vc/cfg/vcaltdb/config/mariadb-community/my.cnf.d/Z-local-3.innodb.cnf

@@ -1,4 +1,3 @@
 [mysqld]
-innodb_buffer_pool_size=5120
 innodb_flush_log_at_trx_commit=1
 innodb_lock_wait_timeout=900

clusters/vc/cfg/vcaltdb/config/mariadb-enterprise/my.cnf.d/Z-local-3.innodb.cnf

@@ -1,4 +1,3 @@
 [mysqld]
-innodb_buffer_pool_size=5120
 innodb_flush_log_at_trx_commit=1
 innodb_lock_wait_timeout=900

clusters/vc/cfg/vcbuild/build/slurm-spank-lua/1.setup-prereqs

@@ -25,6 +25,8 @@ if [ -n "${OUT}" ] ; then
   exec > >(tee -a "${OUT}") 2>&1
 fi
 
+set -xv
+
 HOSTNAME=${HOSTNAME:-$(hostname -s)}
 CLUSTERNAME=${HOSTNAME:0:2}
 XFR=/${CLUSTERNAME}/xfr
@@ -71,10 +73,25 @@ if [ ! -d "${BUILDWHERE_PRODUCT}" ] ; then
 fi
 if [ ! -r "${SLURM_SPANK_LUA_SPEC}" ] ; then
   need_ingest=true
+else
+  SLURM_SPANK_LUA_SPEC_VERSION=$(awk '/Version:/ {print $2}' ${SLURM_SPANK_LUA_SPEC})
+  if [ "${SLURM_SPANK_LUA_SPEC_VERSION}" != "${SLURM_SPANK_LUA_VERSION}" ] ; then
+    Verbose " SLURM_SPANK_LUA_VERSION: ${SLURM_SPANK_LUA_VERSION}"
+    Verbose " SLURM_SPANK_LUA_SPEC_VERSION: ${SLURM_SPANK_LUA_SPEC_VERSION}"
+    ErrExit ${EX_OSFILE} "Update VERSION_FILE to: ${SLURM_SPANK_LUA_SPEC_VERSION}"
+  fi
 fi
 
 if [ -n "${need_ingest}" ] ; then
   Verbose " recreating BUILDWHERE_PRODUCT:${BUILDWHERE_PRODUCT}"
+  if [ -L "${BUILDWHERE_PRODUCT}" ] ; then
+    target=$(readlink -e ${BUILDWHERE_PRODUCT})
+    Verbose "  BUILDWHERE_PRODUCT -> ${target}"
+    if [ "${target}" != "${BUILDWHERE_PRODUCT_VERSION}" ] ; then
+      Verbose "  target:${target} != ${BUILDWHERE_PRODUCT_VERSION} -- version mismatch?"
+      Rc ErrExit ${EX_OSFILE} "rm -f ${BUILDWHERE_PRODUCT_VERSION}"
+    fi
+  fi
   if [ -r ${XFR}/${SLURM_SPANK_LUA_TARBALL} ] ; then
     if [ ! -d ${BUILDWHERE_PRODUCT} ] ; then
       Rc ErrExit ${EX_OSFILE} "tar -xzf ${XFR}/${SLURM_SPANK_LUA_TARBALL} -C ${BUILDWHERE}"
@@ -128,4 +145,5 @@ fi
 Rc ErrExit ${EX_OSFILE} "cp -v --preserve=all ${RPMBUILD}/SOURCES/${slurm_spank_lua_vers}${TAR_SUFFIX} ${XFR}"
 
 exit ${EX_OK}
-# vim: background=dark
+
+# vim: background=dark ts=2 sw=2 syntax=sh

clusters/vc/cfg/vcbuild/build/slurm-spank-lua/version

@@ -1 +1 @@
-0.43
+0.45

clusters/vc/cfg/vcbuild/build/slurm/version

@@ -1 +1 @@
-21.08.2
+22.05.2

clusters/vc/cfg/vcdb/config/mariadb-community/my.cnf.d/Z-local-3.innodb.cnf

@@ -1,4 +1,3 @@
 [mysqld]
-innodb_buffer_pool_size=1280M
 innodb_flush_log_at_trx_commit=1
 innodb_lock_wait_timeout=900

clusters/vc/cfg/vcdb/config/mariadb-enterprise/my.cnf.d/Z-local-3.innodb.cnf

@@ -1,4 +1,3 @@
 [mysqld]
-innodb_buffer_pool_size=1280M
 innodb_flush_log_at_trx_commit=1
 innodb_lock_wait_timeout=900

clusters/vc/cfg/vcdb/config/slurm/3.start-slurmdbd-service

@@ -22,17 +22,20 @@ if [ -n "${OUT}" ] ; then
   exec > >(tee -a "${OUT}") 2>&1
 fi
 
-Rc ErrExit 4 "systemctl start slurmdbd"
-
-isactive=$(systemctl status slurmdbd | grep Active: | awk '{print $2}')
-if [ "${isactive}" != "active" ] ; then
-  ErrExit ${EX_CONFIG} "slurmdbd failed to start correctly: ${isactive}"
+DBDHost=$(grep AccountingStorageHost=  /etc/slurm/slurm.conf | sed 's/AccountingStorageHost=//')
+if [ "${DBDHost}" = $(hostname -s) ] ; then
+	Rc ErrExit 4 "systemctl start slurmdbd"
+
+	isactive=$(systemctl status slurmdbd | grep Active: | awk '{print $2}')
+	if [ "${isactive}" != "active" ] ; then
+		ErrExit ${EX_CONFIG} "slurmdbd failed to start correctly: ${isactive}"
+	fi
+
+	# slurmdbd isn't ready immediately to start servicing requests
+	# if seeing "Connection refused" messages in slurmdbd status, may need to increase this
+	sleep 3
 fi
 
-# slurmdbd isn't ready immediately to start servicing requests
-# if seeing "Connection refused" messages in slurmdbd status, may need to increase this
-sleep 3
-
 Rc ErrExit 3 "sacctmgr show cluster -n"
 
 trap '' 0

clusters/vc/cfg/vcdb/config/slurm/4.create-cluster-db

@@ -70,6 +70,8 @@ do
   fi
 done
 
+ControlHost=$(grep SlurmctldHost /etc/slurm/slurm.conf | sed -s 's/SlurmctldHost=//' | sed 's/(/ /' | sed 's/)/ /' | awk '{print $1}')
+ControlAddr=$(grep SlurmctldHost /etc/slurm/slurm.conf | sed -s 's/SlurmctldHost=//' | sed 's/(/ /' | sed 's/)/ /' | awk '{print $2}')
 exists=$(sacctmgr show cluster ${CLUSTERNAME} -n -P)
 if [ -z "${exists}" ] ; then
   Verbose " creating cluster: ${CLUSTERNAME}"

clusters/vc/cfg/vcdb/config/slurm/5.verify-db

@@ -40,8 +40,14 @@ if [ ! -f ${SLURMDBDCONF} ] ; then
 fi
 
 STORAGEUSER=$(grep StorageUser ${SLURMDBDCONF} | sed 's/StorageUser=//')
+MYSQL_AUTH=""
+if [ -n "${STORAGEUSER}" ] ; then
+	MYSQL_AUTH="--user=${STORAGEUSER} "
+fi
 STORAGEPASS=$(grep StoragePass ${SLURMDBDCONF} | sed 's/StoragePass=//')
-MYSQL_AUTH="--user=${STORAGEUSER} --password=${STORAGEPASS}"
+if [ -n "${STORAGEPASS}" ] ; then
+	MYSQL_AUTH="${MYSQL_AUTH} --password=${STORAGEPASS}"
+fi
 
 export dbdhost=$(grep -i dbdhost ${SLURMDBDCONF} | tr '[:upper:]' '[:lower:]' | sed 's/dbdhost=//')
 export dbdaddr=$(grep -i dbdaddr ${SLURMDBDCONF} | tr '[:upper:]' '[:lower:]' | sed 's/dbdaddr=//')
@@ -53,22 +59,30 @@ if [ -z "${dbdaddr}" ] ; then
   ErrExit ${EX_SOFTWARE} "empty dbdaddr"
 fi
 
-MYSQL_CMD="mysql ${MYSQL_AUTH} --host=${dbdhost} --batch"
+DBDHost=$(grep AccountingStorageHost=  ${SLURMCONF} | sed 's/AccountingStorageHost=//')
 
+MYSQL_CMD="mysql ${MYSQL_AUTH} --batch"
+if [ "${DBDHost}" = $(hostname -s) ] ; then
+	MYSQL_CMD="${MYSQL_CMD} --host=${dbdhost}"
+	grep StorageUser ${SLURMDBDCONF}
+else
+	MYSQL_CMD="${MYSQL_CMD} --host=${DBDHost}"
+	dbdhost=${DBDHost}
+	dbdaddr=$(host ${DBDHost} | awk '{print $4}')
+fi
 Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ping -c 1 -i 1 -n ${dbdhost}"
 Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ping -c 1 -i 1 -n ${dbdaddr}"
-
-grep StorageUser ${SLURMDBDCONF}
-
-Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ${MYSQL_CMD} --execute 'show databases;'"
+#XXX Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ${MYSQL_CMD} --execute 'show databases;'"
+timeout ${TIMEOUT} ${MYSQL_CMD} --execute 'show databases;'
 
 clustername=$(timeout ${TIMEOUT} ${MYSQL_CMD} --database=slurm_acct_db --skip-column-names --execute 'select name from cluster_table;' )
 
-if [[ ${clustername} != ${CLUSTERNAME} ]] ; then
-  ErrExit ${EX_SOFTWARE} "select name from cluster_table => ${clustername} != ${CLUSTERNAME}"
-fi
+#XXX if [[ ${clustername} != ${CLUSTERNAME} ]] ; then
+#XXX   ErrExit ${EX_SOFTWARE} "select name from cluster_table => ${clustername} != ${CLUSTERNAME}"
+#XXX fi
 
-Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ${MYSQL_CMD} --database=slurm_acct_db --skip-column-names --execute 'show tables;' | grep ${CLUSTERNAME}_last_ran"
+#XXX Rc ErrExit ${EX_SOFTWARE}  "timeout ${TIMEOUT} ${MYSQL_CMD} --database=slurm_acct_db --skip-column-names --execute 'show tables;' | grep ${CLUSTERNAME}_last_ran"
+timeout ${TIMEOUT} ${MYSQL_CMD} --database=slurm_acct_db --skip-column-names --execute 'show tables;' | grep ${CLUSTERNAME}_last_ran
 
 trap '' 0
 exit ${EX_OK}

clusters/vc/cfg/vcdb/config/slurm/required.service

@@ -1,2 +1 @@
 munge
-slurmdbd

clusters/vc/cfg/vclogin/config/slurm/5.licenses

@@ -62,7 +62,7 @@ do
   clause=" type=license cluster=${cluster} "
   suffix=" count=${count} "
   if [ -n "${existing_licenses}" ] ; then
-    if [[ "${existing_licenses}" != *"${lic}"*"${cluster}" ]] ; then
+    if [[ "${existing_licenses}" = ${lic}*${cluster} ]] ; then
       action="update"
       clause=" cluster=${cluster} set "
       suffix="" 
@@ -83,10 +83,11 @@ do
       attr_val=$(cat ${d}/${attr_name})
     fi
     if [[ "${existing}" == *"${lic_name}"* ]] ; then
-	existing_val=$(sacctmgr show resource withcluster where cluster=${cluster} name=${lic_name} -n | awk '{print $5}') 
+	existing_val=$(sacctmgr show resource withcluster where cluster=${cluster} name=${lic_name} -n | awk '{print $7}') 
 	if [ "${existing_val}" = "${attr_val}" ] ; then
 	  if [ "${attr_val}" -ne 0 ] ; then
             Verbose "license=${lic_name}, cluster=${cluster}, existing_val:${existing_val} = attr_val:${attr_val}, skipped"
+
             continue
 	  fi
 	fi

clusters/vc/cfg/vclogin/config/slurm/ipaddr-slurmdbd.%VCDB%

@@ -1 +1 @@
-%VCDB%
+192.168.56.1

clusters/vc/cfg/vcsvc/config/chrony/3.chrony-waitsync

@@ -21,7 +21,7 @@ if [ -n "${OUT}" ] ; then
 fi
 SetFlags >/dev/null 2>&1
 
-( cd / ; chronyc waitsync ) &
+( cd / ; timeout 10 chronyc waitsync ) &
 
 trap '' 0
 exit ${EX_OK}

clusters/vc/common/etc/hosts.%VC-NET%

@@ -7,12 +7,14 @@
 
 ## hostonly network: 
 
+#10.0.2.2	gw.host-local gw host-gw gw.host ext-db db-ext db.host db.host-local db-host-local db
+
 ### following is used by generateproviderfiles.sh
 ### 192.168.56 %%%VC-VIRTUALBOX-NET%%% ###
 ### 192.168.56 %%%VX-VIRTUALBOX-NET%%% ###
 
 192.168.56.0   virtualbox-net0.tc hostonly.tc hostonly hostonly.test-cluster hostonly.testcluster vbox-net
-192.168.56.1   gw.vbox.tc gw-vbox gw.vbox.test-cluster gw.vbox.testcluster
+192.168.56.1   gw.vbox.tc gw-vbox gw.vbox.test-cluster gw.vbox.testcluster db-host-local db.host-local db.host db
 
 ### following is used by generateproviderfiles.sh
 ### 192.168.78 %%%VC-LIBVIRT-NET%%% ###

clusters/vc/common/etc/networks.%VC-NET%

@@ -1,5 +1,12 @@
 default				0.0.0.0
 loopback			127.0.0.0
+loopback-netmask		255.0.0.0
+
 link-local			169.254.0.0
+link-local-netmask		255.255.0.0
+
 vc-net				%VC-0NET%	virtual-cluster-net virtualcluster-net test-cluster-net vx-net
+vc-netmask			255.255.255.0
+
 nat-ext-net			10.0.2.0	nat-external
+nat-ext-netmask			255.255.255.0

clusters/vc/common/etc/slurm/slurm.conf.template.%VCSCHED%

@@ -1,4 +1,5 @@
 SlurmctldHost=vcsched(%VCSCHED%)
+SlurmctldAddr=%VCSCHED%
 AuthType=auth/munge
 CryptoType=crypto/munge
 DisableRootJobs=NO
@@ -46,6 +47,7 @@ ReturnToService=2
 SallocDefaultCommand="srun -n1 -N1 --mem-per-cpu=0 --pty --preserve-env $SHELL"
 SlurmctldPidFile=/var/run/slurmctld.pid
 SlurmctldPort=6817
+#SlurmctldParameters=preempt_send_user_signal,max_dbd_msg_action=exit #schedmd#14358
 SlurmctldParameters=preempt_send_user_signal
 SlurmdPidFile=/var/run/slurm/slurmd.pid
 SlurmdPort=6818
@@ -121,16 +123,16 @@ PriorityWeightQOS=86400
 #
 # LOGGING AND ACCOUNTING
 AccountingStorageEnforce=associations,limits,qos,safe
-AccountingStorageHost=vcdb
-AccountingStorageExternalHost=vxdb
+AccountingStorageHost=db-host-local
+#AccountingStorageExternalHost=db-host-local
 AccountingStorageLoc=slurm_acct_db
 #AccountingStoragePass=
 #AccountingStoragePort=
 AccountingStorageType=accounting_storage/slurmdbd
 #AccountingStorageUser=
 #pre21.08 AccountingStoreJobComment=YES
-AccountingStoreFlags=job_comment
-# MaxDBDMsgs=Max(MaxJobCount * 2 + Node Count * 4, 10,000)
+AccountingStoreFlags=job_comment,job_env,job_script
+# default: MaxDBDMsgs=Max(MaxJobCount * 2 + Node Count * 4, 10,000)
 MaxDBDMsgs=200000
 ClusterName=vc
 DebugFlags=Agent,Backfill,BackfillMap,CPU_Bind,Dependency,License,Network,NodeFeatures,Power,Priority,Reservation,Steps,TraceJobs,Triggers,WorkQueue
@@ -195,6 +197,6 @@ PartitionName=login     Nodes=vclogin				        Default=NO   State=UP
 PartitionName=compile   Nodes=vcbuild				        Default=NO   State=UP   	                    MaxTime=1-00:00:00 	PriorityTier=10
 PartitionName=exclusive Nodes=vc[1-2] Oversubscribe=EXCLUSIVE	        Default=NO   State=UP	DefaultTime=1:00:00 MaxTime=10-00:00:00	PriorityTier=100
 PartitionName=shared    Nodes=vc[1-2] Oversubscribe=FORCE   	        Default=YES  State=UP	DefaultTime=1:00:00 MaxTime=90-00:00:00	PriorityTier=1000
-PartitionName=infrastructure Nodes=vcfs,vcsvc,vcdb,vcaltdb,vcsched,vcgate Default=NO State=Down	DefaultTime=0:00:01 MaxTime=00:00:01    PriorityTier=0
+PartitionName=infrastructure Nodes=vcfs,vcsvc,vcdb,vcaltdb,vcsched,vcgate Default=NO State=Down	DefaultTime=0:00:01 MaxTime=00:00:01    PriorityTier=0 DenyAccounts=ALL DenyQOS=ALL Hidden=YES
 
 

clusters/vc/common/etc/slurm/slurmdbd.conf.%VCDB%

@@ -1,5 +1,5 @@
-DbdAddr=%VCDB%
-DbdHost=vcdb
+DbdAddr=192.168.56.1
+DbdHost=db-host-local
 SlurmUser=root
 DebugFlags=DB_ASSOC,DB_EVENT,DB_QOS,DB_JOB
 DebugLevel=debug3

clusters/vc/common/home/sts/.login.turq-slurmdbta

@@ -0,0 +1,18 @@
+setenv http_proxy http://proxyout.lanl.gov:8080
+setenv https_proxy http://proxyout.lanl.gov:8080
+setenv proxy http://proxyout.lanl.gov:8080
+
+setenv SLURM_CONF /etc/slurm/slurm.conf.vc
+
+pushd ~/hpc-collab/http-repo
+source bin/setpath.csh
+mkdir -p /tmp/tarballs
+mkdir -p /tmp/VirtualBox\ VMs
+pushd clusters/vc
+make Vagrantfile
+cd ../vx
+make Vagrantfile
+show
+popd
+
+