Merge pull request #97 from jkaninda/multi-backup

Add multi database backup

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           push: true
-          file: "./docker/Dockerfile"
+          file: "./Dockerfile"
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           build-args: |
             appVersion=develop-${{ github.sha }}

.github/workflows/release.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           push: true
-          file: "./docker/Dockerfile"
+          file: "./Dockerfile"
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           build-args: |
             appVersion=${{ env.TAG_NAME }}

docs/how-tos/deprecated-configs.md

@@ -0,0 +1,6 @@
+---
+title: Update deprecated configurations
+layout: default
+parent: How Tos
+nav_order: 11
+---

docs/how-tos/mutli-backup.md

@@ -0,0 +1,40 @@
+---
+title: Run multiple database backup schedules in the same container
+layout: default
+parent: How Tos
+nav_order: 11
+---
+
+Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml`  `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`.
+
+## Configuration file
+
+```yaml
+#cronExpression: "@every 20m" //Optional, for scheduled backups
+cronExpression: "" 
+databases:
+  - host: postgres1
+    port: 5432
+    name: database1
+    user: database1
+    password: password
+    path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: postgres2
+    port: 5432
+    name: lldap
+    user: lldap
+    password: password
+    path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: postgres3
+    port: 5432
+    name: keycloak
+    user: keycloak
+    password: password
+    path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: postgres4
+    port: 5432
+    name: joplin
+    user: joplin
+    password: password
+    path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/)
+```

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/crypto v0.28.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

pkg/backup.go

@@ -20,23 +20,28 @@ import (
 
 func StartBackup(cmd *cobra.Command) {
 	intro()
-	dbConf = initDbConfig(cmd)
 	//Initialize backup configs
 	config := initBackupConfig(cmd)
-
-	if config.cronExpression == "" {
-		BackupTask(dbConf, config)
-	} else {
-		if utils.IsValidCronExpression(config.cronExpression) {
-			scheduledMode(dbConf, config)
+	//Load backup configuration file
+	configFile, err := loadConfigFile()
+	if err != nil {
+		dbConf = initDbConfig(cmd)
+		if config.cronExpression == "" {
+			BackupTask(dbConf, config)
 		} else {
-			utils.Fatal("Cron expression is not valid: %s", config.cronExpression)
+			if utils.IsValidCronExpression(config.cronExpression) {
+				scheduledMode(dbConf, config)
+			} else {
+				utils.Fatal("Cron expression is not valid: %s", config.cronExpression)
+			}
 		}
+	} else {
+		startMultiBackup(config, configFile)
 	}
 
 }
 
-// Run in scheduled mode
+// scheduledMode Runs backup in scheduled mode
 func scheduledMode(db *dbConfig, config *BackupConfig) {
 	utils.Info("Running in Scheduled mode")
 	utils.Info("Backup cron expression:  %s", config.cronExpression)
@@ -63,6 +68,17 @@ func scheduledMode(db *dbConfig, config *BackupConfig) {
 	defer c.Stop()
 	select {}
 }
+
+// multiBackupTask backup multi database
+func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
+	for _, db := range databases {
+		//Check if path is defined in config file
+		if db.Path != "" {
+			bkConfig.remotePath = db.Path
+		}
+		BackupTask(getDatabase(db), bkConfig)
+	}
+}
 func BackupTask(db *dbConfig, config *BackupConfig) {
 	utils.Info("Starting backup task...")
 	//Generate file name
@@ -85,9 +101,54 @@ func BackupTask(db *dbConfig, config *BackupConfig) {
 		localBackup(db, config)
 	}
 }
-func intro() {
-	utils.Info("Starting PostgreSQL Backup...")
-	utils.Info("Copyright (c) 2024 Jonas Kaninda ")
+func startMultiBackup(bkConfig *BackupConfig, configFile string) {
+	utils.Info("Starting multiple backup job...")
+	var conf = &Config{}
+	conf, err := readConf(configFile)
+	if err != nil {
+		utils.Fatal("Error reading config file: %s", err)
+	}
+	//Check if cronExpression is defined in config file
+	if conf.CronExpression != "" {
+		bkConfig.cronExpression = conf.CronExpression
+	}
+	// Check if cronExpression is defined
+	if bkConfig.cronExpression == "" {
+		multiBackupTask(conf.Databases, bkConfig)
+	} else {
+		// Check if cronExpression is valid
+		if utils.IsValidCronExpression(bkConfig.cronExpression) {
+			utils.Info("Running MultiBackup in Scheduled mode")
+			utils.Info("Backup cron expression:  %s", bkConfig.cronExpression)
+			utils.Info("Storage type %s ", bkConfig.storage)
+
+			//Test backup
+			utils.Info("Testing backup configurations...")
+			multiBackupTask(conf.Databases, bkConfig)
+			utils.Info("Testing backup configurations...done")
+			utils.Info("Creating multi backup job...")
+			// Create a new cron instance
+			c := cron.New()
+
+			_, err := c.AddFunc(bkConfig.cronExpression, func() {
+				// Create a channel
+				multiBackupTask(conf.Databases, bkConfig)
+			})
+			if err != nil {
+				return
+			}
+			// Start the cron scheduler
+			c.Start()
+			utils.Info("Creating multi backup job...done")
+			utils.Info("Backup job started")
+			defer c.Stop()
+			select {}
+
+		} else {
+			utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression)
+		}
+	}
+
 }
 
 // BackupDatabase backup database
@@ -119,7 +180,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 			log.Fatal(err)
 		}
 		// save output
-		file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
+		file, err := os.Create(filepath.Join(tmpPath, backupFileName))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -145,7 +206,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		gzipCmd := exec.Command("gzip")
 		gzipCmd.Stdin = stdout
 		// save output
-		gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
+		gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName))
 		gzipCmd.Start()
 		if err != nil {
 			log.Fatal(err)
@@ -180,6 +241,7 @@ func localBackup(db *dbConfig, config *BackupConfig) {
 	}
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
 }
 
 func s3Backup(db *dbConfig, config *BackupConfig) {
@@ -220,6 +282,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
+
 }
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
@@ -255,6 +319,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
+
 }
 func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
@@ -290,6 +356,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
 }
 
 func encryptBackup(config *BackupConfig) {

pkg/config.go

@@ -14,7 +14,17 @@ import (
 	"strconv"
 )
 
+type Database struct {
+	Host     string `yaml:"host"`
+	Port     string `yaml:"port"`
+	Name     string `yaml:"name"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	Path     string `yaml:"path"`
+}
 type Config struct {
+	Databases      []Database `yaml:"databases"`
+	CronExpression string     `yaml:"cronExpression"`
 }
 
 type dbConfig struct {
@@ -92,6 +102,16 @@ func initDbConfig(cmd *cobra.Command) *dbConfig {
 	return &dConf
 }
 
+func getDatabase(database Database) *dbConfig {
+	return &dbConfig{
+		dbHost:     database.Host,
+		dbPort:     database.Port,
+		dbName:     database.Name,
+		dbUserName: database.User,
+		dbPassword: database.Password,
+	}
+}
+
 // loadSSHConfig loads the SSH configuration from environment variables
 func loadSSHConfig() (*SSHConfig, error) {
 	utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME")
@@ -245,3 +265,10 @@ func initTargetDbConfig() *targetDbConfig {
 	}
 	return &tdbConfig
 }
+func loadConfigFile() (string, error) {
+	backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE"))
+	if err == nil {
+		return backupConfigFile, nil
+	}
+	return "", fmt.Errorf("backup config file not found")
+}

pkg/helper.go

@@ -10,12 +10,18 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/jkaninda/pg-bkup/utils"
+	"gopkg.in/yaml.v3"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
 )
 
+func intro() {
+	utils.Info("Starting PostgreSQL Backup...")
+	utils.Info("Copyright (c) 2024 Jonas Kaninda ")
+}
+
 // copyToTmp copy file to temporary directory
 func copyToTmp(sourcePath string, backupFileName string) {
 	//Copy backup from storage to /tmp
@@ -139,6 +145,8 @@ func testDatabaseConnection(db *dbConfig) {
 	utils.Info("Successfully connected to %s database", db.dbName)
 
 }
+
+// checkPubKeyFile checks gpg public key
 func checkPubKeyFile(pubKey string) (string, error) {
 	// Define possible key file names
 	keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey}
@@ -160,6 +168,8 @@ func checkPubKeyFile(pubKey string) (string, error) {
 	// Return an error if neither file exists
 	return "", fmt.Errorf("no public key file found")
 }
+
+// checkPrKeyFile checks private key
 func checkPrKeyFile(prKey string) (string, error) {
 	// Define possible key file names
 	keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey}
@@ -181,3 +191,45 @@ func checkPrKeyFile(prKey string) (string, error) {
 	// Return an error if neither file exists
 	return "", fmt.Errorf("no public key file found")
 }
+
+// readConf reads config file and returns Config
+func readConf(configFile string) (*Config, error) {
+	if utils.FileExists(configFile) {
+		buf, err := os.ReadFile(configFile)
+		if err != nil {
+			return nil, err
+		}
+
+		c := &Config{}
+		err = yaml.Unmarshal(buf, c)
+		if err != nil {
+			return nil, fmt.Errorf("in file %q: %w", configFile, err)
+		}
+
+		return c, err
+	}
+	return nil, fmt.Errorf("config file %q not found", configFile)
+}
+
+// checkConfigFile checks config files and returns one config file
+func checkConfigFile(filePath string) (string, error) {
+	// Define possible config file names
+	configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath}
+
+	// Loop through config file names and check if they exist
+	for _, configFile := range configFiles {
+		if _, err := os.Stat(configFile); err == nil {
+			// File exists
+			return configFile, nil
+		} else if os.IsNotExist(err) {
+			// File does not exist, continue to the next one
+			continue
+		} else {
+			// An unexpected error occurred
+			return "", err
+		}
+	}
+
+	// Return an error if neither file exists
+	return "", fmt.Errorf("no config file found")
+}

pkg/var.go

@@ -16,6 +16,7 @@ var (
 	file    = ""
 
 	storagePath        = "/backup"
+	workingDir         = "/config"
 	disableCompression = false
 	encryption         = false
 	usingKey           = false

utils/utils.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 )
 
+// FileExists checks if the file does exist
 func FileExists(filename string) bool {
 	info, err := os.Stat(filename)
 	if os.IsNotExist(err) {