From 15b829b77ef0ef0e7766d7d18f508f14538ba039 Mon Sep 17 00:00:00 2001 From: Arjun Kondur Date: Thu, 1 Aug 2024 14:26:30 -0500 Subject: [PATCH] Update documentation, comments --- docs/SplunkOperatorUpgrade.md | 24 ++++++++++++------------ pkg/splunk/enterprise/upgrade.go | 17 +++++++++++++---- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/SplunkOperatorUpgrade.md b/docs/SplunkOperatorUpgrade.md index 9e276ad12..9f4c030e6 100644 --- a/docs/SplunkOperatorUpgrade.md +++ b/docs/SplunkOperatorUpgrade.md @@ -142,18 +142,18 @@ kubectl get pods splunk--monitoring-console-0 -o yaml | grep -i image image: splunk/splunk:9.1.3 imagePullPolicy: IfNotPresent ``` -## Splunk Enterprise Cluster upgrade example +## Splunk Enterprise Cluster upgrade -This is an example of the process followed by the Splunk Operator if the operator version is upgraded and a later Splunk Enterprise Docker image is available: -​ +The Splunk Operator mostly adheres to the upgrade path steps delineated in the Splunk documentation. All pods of the custom resources are deleted and redeployed sequentially. In cases where multi-zone Indexer clusters are utilized, they undergo redeployment zone by zone. Each pod upgrade is meticulously verified to ensure a successful process, with thorough checks conducted to confirm that everything is functioning as expected. If there are multiple pods per Custom Resource, the pods are terminated and re-deployed in a descending order with the highest numbered pod going first. + +This is an example of the process followed by the Splunk Operator if the operator version is upgraded and a later Splunk Enterprise Docker image is available. Pod termination and redeployment occur in the below mentioned order based on the recommended upgrade path: -1. Initiation of a new Splunk Operator pod will lead to the termination of the existing operator pod. -2. All existing License Manager, Standalone, Monitoring Console, Cluster Manager, Search Head, ClusterManager, and Indexer pods will undergo termination for subsequent redeployment with upgraded specifications. -3. The Splunk Operator adheres to the upgrade path steps delineated in the Splunk documentation. Pod termination and redeployment occur in a specific order based on the recommended upgrade path. -4. Standalone or License manager will be the first to be redeployed -5. Next ClusterManager pod will be redeployed, next the Monitoring Console pod undergoes termination and redeployment. -6. Subsequently, the Search Head cluster pods connected to it are terminated and redeployed. -7. Afterwards, all pods in the Indexer cluster are redeployed sequentially. In cases where multi-zone Indexer clusters are utilized, they undergo redeployment zone by zone. -8. Each pod upgrade is meticulously verified to ensure a successful process, with thorough checks conducted to confirm that everything is functioning as expected. +1. Splunk Operator deployment pod +2. Standalone +3. License manager +4. ClusterManager +5. Search Head cluster +6. Indexer Cluster +7. Monitoring Console -* Note: If there are multiple pods per Custom Resource, the pods are terminated and re-deployed in a descending order with the highest numbered pod going first +Note: The order above assumes that the custom resources are linked via references. If there are Custom resources without references they will be deleted/redeployed indepedentlty of the order. diff --git a/pkg/splunk/enterprise/upgrade.go b/pkg/splunk/enterprise/upgrade.go index 0ca1ec1cb..21bc8dd2b 100644 --- a/pkg/splunk/enterprise/upgrade.go +++ b/pkg/splunk/enterprise/upgrade.go @@ -3,6 +3,7 @@ package enterprise import ( "context" "fmt" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -259,14 +260,16 @@ MonitoringConsole: runtime.InNamespace(cr.GetNamespace()), } + // get the list of cluster managers clusterManagerList := &enterpriseApi.ClusterManagerList{} - // get the list cluster manager which have mc reference err := c.List(ctx, clusterManagerList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Cluster Manager list. Reason %v", err)) scopedLog.Error(err, "Unable to get clusterManager list") return false, err } + + // Run through list, if it has the MC reference, bail out if it is NOT ready for _, cm := range clusterManagerList.Items { if cm.Spec.MonitoringConsoleRef.Name == cr.GetName() { if cm.Status.Phase != enterpriseApi.PhaseReady { @@ -276,8 +279,8 @@ MonitoringConsole: } } + // get the list of search head clusters searchHeadClusterList := &enterpriseApi.SearchHeadClusterList{} - // get the list search head which have mc reference err = c.List(ctx, searchHeadClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Search Head Cluster list. Reason %v", err)) @@ -285,6 +288,7 @@ MonitoringConsole: return false, err } + // Run through list, if it has the MC reference, bail out if it is NOT ready for _, shc := range searchHeadClusterList.Items { if shc.Spec.MonitoringConsoleRef.Name == cr.GetName() { if shc.Status.Phase != enterpriseApi.PhaseReady { @@ -293,8 +297,9 @@ MonitoringConsole: } } } + + // get the list of indexer clusters indexerClusterList := &enterpriseApi.IndexerClusterList{} - // get the list indexer which have mc reference err = c.List(ctx, indexerClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Indexer list. Reason %v", err)) @@ -302,6 +307,7 @@ MonitoringConsole: return false, err } + // Run through list, if it has the MC reference, bail out if it is NOT ready for _, idx := range indexerClusterList.Items { if idx.Name == cr.GetName() { if idx.Status.Phase != enterpriseApi.PhaseReady { @@ -310,14 +316,17 @@ MonitoringConsole: } } } + + // get the list of standalones standaloneList := &enterpriseApi.IndexerClusterList{} - // get the list standalone which have mc reference err = c.List(ctx, standaloneList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Standalone list. Reason %v", err)) scopedLog.Error(err, "Unable to get standalone list") return false, err } + + // Run through list, if it has the MC reference, bail out if it is NOT ready for _, stdln := range standaloneList.Items { if stdln.Name == cr.GetName() { if stdln.Status.Phase != enterpriseApi.PhaseReady {