Merge pull request #13 from klouddb/release-v2.2

Enhance backup audit tool management and PostgreSQL configuration

.github/workflows/postgresconf.yml

@@ -0,0 +1,114 @@
+name: postgresconf
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  postgresconf:
+    name: "Postgres version : ${{ matrix.postgres_version }}, Testcase : ${{ matrix.testcases }}" 
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        postgres_version: [13,14,15,16]
+        testcases: [9]
+      max-parallel: 4
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.1
+          cache: true
+      - name: make install
+        run: make install
+      - name: Create temp directory with random name
+        run: |
+          cd docker_testing/postgresconfig
+          RANDOM_NUM=$((RANDOM % 10000))
+          TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+          TEMP_DIR="$PWD/temp_${TIMESTAMP}_${RANDOM_NUM}"
+          mkdir -p "$TEMP_DIR"
+          echo "Temp directory created: $TEMP_DIR"
+          echo "TEMP_DIR=$TEMP_DIR" >> $GITHUB_ENV
+      - name: generate postgres config
+        run: |
+          cd $TEMP_DIR
+          ciscollector -r -config=.. < ../input-${{ matrix.postgres_version }}-${{ matrix.testcases }}.txt
+      - name: print postgres config
+        run: |
+          cat $TEMP_DIR/postgresql.conf
+      - name: Set container name
+        run: |
+          echo "CONTAINER_NAME=postgres-test-${{ matrix.postgres_version }}-${{ matrix.testcases }}" >> $GITHUB_ENV
+      - name: Run PostgreSQL Container
+        run: |
+          echo "Container name : ${CONTAINER_NAME}"
+          docker run --name $CONTAINER_NAME \
+          -e POSTGRES_PASSWORD=mysecretpassword \
+          -d postgres:${{ matrix.postgres_version }}
+      - name: Wait for PostgreSQL to be ready
+        run: sleep 2
+      - name: Copy Custom Config
+        run: |
+          docker cp $TEMP_DIR/postgresql.conf $CONTAINER_NAME:/var/lib/postgresql/data/postgresql.conf
+      - name: Wait for PostgreSQL to be ready after config
+        run: sleep 2
+      - name: Reload PostgreSQL with New Config without restart
+        run: |
+          docker exec $CONTAINER_NAME \
+          psql -U postgres -c "SELECT pg_reload_conf();"
+      - name: Run Validation Script after reload
+        run: |
+          errors=$(docker exec $CONTAINER_NAME \
+            psql -U postgres -t -c "SELECT COUNT(*) FROM pg_file_settings WHERE error IS NOT NULL;")
+
+          if [ "$errors" -ne 0 ]; then
+            echo -e "\033[0;31mSome configuration settings require restart.\033[0m"
+
+            docker exec $CONTAINER_NAME \
+              psql -U postgres -c "SELECT * FROM pg_file_settings WHERE error IS NOT NULL;"
+          else
+            echo -e "\033[0;32mNo configuration errors found.\033[0m"
+            echo -e "\033[0;32mApplied configuration is.\033[0m"
+            docker exec $CONTAINER_NAME \
+              psql -U postgres -c "SELECT * FROM pg_file_settings;"
+          fi
+      - name: Restart PostgreSQL Container
+        run: |
+          docker restart $CONTAINER_NAME
+      - name: Wait for PostgreSQL to be ready after restart
+        run: sleep 2
+      - name: Run Validation Script after restart
+        run: |
+          container_status=$(docker inspect -f '{{.State.Running}}' $CONTAINER_NAME 2>/dev/null || echo "not_found")
+
+          if [ "$container_status" != "true" ]; then
+            echo "Container is not running or not found. Attempting to print logs."
+            docker logs $CONTAINER_NAME || echo "No logs found or unable to retrieve logs."
+            exit 1
+          fi
+
+          errors=$(docker exec $CONTAINER_NAME \
+            psql -U postgres -t -c "SELECT COUNT(*) FROM pg_file_settings WHERE error IS NOT NULL;")
+
+          if [ "$errors" -ne 0 ]; then
+            echo "Container logs"
+            docker logs $CONTAINER_NAME
+
+            echo -e "\033[0;31mConfiguration errors found in pg_file_settings.\033[0m"
+
+            docker exec $CONTAINER_NAME \
+              psql -U postgres -c "SELECT * FROM pg_file_settings WHERE error IS NOT NULL;"
+            exit 1
+          else
+            echo -e "\033[0;32mNo configuration errors found.\033[0m"
+            echo -e "\033[0;32mApplied configuration is.\033[0m"
+            docker exec $CONTAINER_NAME \
+              psql -U postgres -c "SELECT * FROM pg_file_settings;"
+          fi

cmd/ciscollector/backuphistory.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sort"
+	"time"
+
+	"github.com/klouddb/klouddbshield/htmlreport"
+	"github.com/klouddb/klouddbshield/pkg/backuphistory"
+)
+
+type backupHistory struct {
+	backupHistoryInput backuphistory.BackupHistoryInput
+	htmlReportHelper   *htmlreport.HtmlReportHelper
+}
+
+func newBackupHistory(backupHistoryInput backuphistory.BackupHistoryInput, htmlReportHelper *htmlreport.HtmlReportHelper) *backupHistory {
+	return &backupHistory{
+		backupHistoryInput: backupHistoryInput,
+		htmlReportHelper:   htmlReportHelper,
+	}
+}
+
+func (h *backupHistory) cronProcess(ctx context.Context) error {
+	return h.run(ctx)
+}
+
+func (h *backupHistory) run(_ context.Context) error {
+	if h.backupHistoryInput.BackupFrequency == "" {
+		return fmt.Errorf("backup frequency is required")
+	}
+
+	var backupHistory []time.Time
+	var err error
+
+	switch h.backupHistoryInput.BackupTool {
+	case "pgbackrest":
+		backupHistory, err = backuphistory.GetBackupHistoryForPgBackrest()
+		if err != nil {
+			return err
+		}
+	case "pg_dump", "pg_dumpall", "pg_basebackup":
+		backupHistory, err = backuphistory.GetBackupHistory(h.backupHistoryInput.BackupPath)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unsupported backup tool: %s", h.backupHistoryInput.BackupTool)
+	}
+
+	if len(backupHistory) == 0 {
+		return fmt.Errorf("no backup history found")
+	} else if len(backupHistory) == 1 {
+		return fmt.Errorf("only one backup history found")
+	}
+
+	sort.Slice(backupHistory, func(i, j int) bool {
+		return backupHistory[i].After(backupHistory[j])
+	})
+
+	missingDates := []string{}
+	previousDate := backupHistory[len(backupHistory)-1]
+
+	if h.backupHistoryInput.BackupFrequency == "daily" {
+		for i := len(backupHistory) - 2; i >= 0; i-- {
+			if previousDate.Format(time.DateOnly) == backupHistory[i].Format(time.DateOnly) {
+				continue
+			}
+
+			nextDate := previousDate.AddDate(0, 0, 1)
+			for nextDate.Format(time.DateOnly) != backupHistory[i].Format(time.DateOnly) {
+				missingDates = append(missingDates, nextDate.Format(time.DateOnly))
+				nextDate = nextDate.AddDate(0, 0, 1)
+			}
+
+			previousDate = backupHistory[i]
+		}
+	} else if h.backupHistoryInput.BackupFrequency == "weekly" {
+		for i := len(backupHistory) - 2; i >= 0; i-- {
+			previousYear, previousWeek := previousDate.ISOWeek()
+			currentYear, currentWeek := backupHistory[i].ISOWeek()
+
+			if previousYear == currentYear && previousWeek == currentWeek {
+				continue
+			}
+
+			nextDate := previousDate.AddDate(0, 0, 7)
+			nextDateYear, nextDateWeek := nextDate.ISOWeek()
+			for nextDateYear != currentYear || nextDateWeek != currentWeek {
+				missingDates = append(missingDates, fmt.Sprintf("year %d - week %d", nextDateYear, nextDateWeek))
+				nextDate = nextDate.AddDate(0, 0, 7)
+				nextDateYear, nextDateWeek = nextDate.ISOWeek()
+			}
+
+			previousDate = backupHistory[i]
+		}
+	} else if h.backupHistoryInput.BackupFrequency == "monthly" {
+		for i := len(backupHistory) - 2; i >= 0; i-- {
+			if previousDate.Format("2006-01") == backupHistory[i].Format("2006-01") {
+				continue
+			}
+
+			nextDate := previousDate.AddDate(0, 1, 0)
+			for nextDate.Format("2006-01") != backupHistory[i].Format("2006-01") {
+				missingDates = append(missingDates, nextDate.Format("2006-01"))
+				nextDate = nextDate.AddDate(0, 1, 0)
+			}
+
+			previousDate = backupHistory[i]
+		}
+	} else {
+		return fmt.Errorf("unsupported backup frequency: %s", h.backupHistoryInput.BackupFrequency)
+	}
+
+	uniqueMissingDates := []string{}
+	for _, v := range missingDates {
+		if !slices.Contains(uniqueMissingDates, v) {
+			uniqueMissingDates = append(uniqueMissingDates, v)
+		}
+	}
+
+	output := backuphistory.BackupHistoryOutput{
+		MissingDates:    uniqueMissingDates,
+		StartDate:       backupHistory[len(backupHistory)-1].Format("2006-01-02"),
+		EndDate:         backupHistory[0].Format("2006-01-02"),
+		BackupFrequency: h.backupHistoryInput.BackupFrequency,
+	}
+
+	backuphistory.PrintBackupHistory(output)
+
+	h.htmlReportHelper.RegisterBackupHistory(output)
+
+	return nil
+}

cmd/ciscollector/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	_ "net/http/pprof"
 	"os"
 	"strings"
 	"time"
@@ -183,10 +184,14 @@ func main() {
 		}
 	}
 
+	if cnf.BackupHistoryInput.BackupTool != "" {
+		overviewErrorMap[cons.RootCMD_BackupAuditTool] = newBackupHistory(cnf.BackupHistoryInput, htmlReportHelper).run(ctx)
+	}
+
 	if cnf.CreatePostgresConfig {
-		overviewErrorMap[cons.RootCMD_CreatePostgresconfig] = postgresconfig.NewProcessor(".").Run(context.TODO())
-		if overviewErrorMap[cons.RootCMD_CreatePostgresconfig] != nil {
-			fmt.Println("> Error while generating postgresql.conf file: ", text.FgHiRed.Sprint(overviewErrorMap[cons.RootCMD_CreatePostgresconfig]))
+		overviewErrorMap[cons.RootCMD_CreatePostgresConfig] = postgresconfig.NewProcessor(".").Run(context.TODO())
+		if overviewErrorMap[cons.RootCMD_CreatePostgresConfig] != nil {
+			fmt.Println("> Error while generating postgresql.conf file: ", text.FgHiRed.Sprint(overviewErrorMap[cons.RootCMD_CreatePostgresConfig]))
 		}
 	}
 

docker_testing/postgresconfig/input-13-1.txt

@@ -0,0 +1,20 @@
+15
+Y
+13
+4GB
+32
+1
+100GB
+1
+3
+100
+y
+*
+1GB
+60
+-1
+2
+3
+off
+60000
+1800000

docker_testing/postgresconfig/input-13-10.txt

@@ -0,0 +1,13 @@
+5
+13
+8GB
+8
+1
+1
+3
+*
+5432
+3
+1GB
+y
+y

docker_testing/postgresconfig/input-13-2.txt

@@ -0,0 +1,51 @@
+15
+ye
+Ys
+y
+1.4
+13.0
+13
+0GB
+1GB
+4.0
+0
+1
+-1
+4
+1
+0GB
+100GB
+10
+-5
+1
+-100
+-10
+0
+-10
+0
+2
+yess
+randomstring
+y
+first(abc)
+0GB
+1GB
+1ms
+1
+-2
+-10
+-1
+-2
+-10
+-1
+-2
+-10
+1
+onn
+off
+-2
+-10
+0
+-2
+-10
+0