From ad9d7a00cca882f13973b8574de044b94a492046 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 9 Oct 2024 12:05:37 +0200 Subject: [PATCH] feat: add multi database backup --- docker/Dockerfile => Dockerfile | 0 docs/how-tos/deprecated-configs.md | 6 +++++ docs/how-tos/mutli-backup.md | 40 ++++++++++++++++++++++++++++++ pkg/backup.go | 26 +++++++++++-------- pkg/config.go | 3 +-- pkg/helper.go | 10 +++++++- utils/utils.go | 1 + 7 files changed, 72 insertions(+), 14 deletions(-) rename docker/Dockerfile => Dockerfile (100%) create mode 100644 docs/how-tos/deprecated-configs.md create mode 100644 docs/how-tos/mutli-backup.md diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/docs/how-tos/deprecated-configs.md b/docs/how-tos/deprecated-configs.md new file mode 100644 index 0000000..1813ff2 --- /dev/null +++ b/docs/how-tos/deprecated-configs.md @@ -0,0 +1,6 @@ +--- +title: Update deprecated configurations +layout: default +parent: How Tos +nav_order: 11 +--- \ No newline at end of file diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md new file mode 100644 index 0000000..1a033a8 --- /dev/null +++ b/docs/how-tos/mutli-backup.md @@ -0,0 +1,40 @@ +--- +title: Run multiple database backup schedules in the same container +layout: default +parent: How Tos +nav_order: 11 +--- + +Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml` `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`. + +## Configuration file + +```yaml +#cronExpression: "@every 20m" //Optional, for scheduled backups +cronExpression: "" +databases: + - host: postgres1 + port: 5432 + name: database1 + user: database1 + password: password + path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres2 + port: 5432 + name: lldap + user: lldap + password: password + path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres3 + port: 5432 + name: keycloak + user: keycloak + password: password + path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres4 + port: 5432 + name: joplin + user: joplin + password: password + path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/) +``` \ No newline at end of file diff --git a/pkg/backup.go b/pkg/backup.go index bb417f5..6851564 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -24,9 +24,7 @@ func StartBackup(cmd *cobra.Command) { config := initBackupConfig(cmd) //Load backup configuration file configFile, err := loadConfigFile() - if err == nil { - startMultiBackup(config, configFile) - } else { + if err != nil { dbConf = initDbConfig(cmd) if config.cronExpression == "" { BackupTask(dbConf, config) @@ -37,11 +35,13 @@ func StartBackup(cmd *cobra.Command) { utils.Fatal("Cron expression is not valid: %s", config.cronExpression) } } + } else { + startMultiBackup(config, configFile) } } -// Run in scheduled mode +// scheduledMode Runs backup in scheduled mode func scheduledMode(db *dbConfig, config *BackupConfig) { utils.Info("Running in Scheduled mode") utils.Info("Backup cron expression: %s", config.cronExpression) @@ -68,6 +68,8 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { defer c.Stop() select {} } + +// multiBackupTask backup multi database func multiBackupTask(databases []Database, bkConfig *BackupConfig) { for _, db := range databases { //Check if path is defined in config file @@ -100,7 +102,7 @@ func BackupTask(db *dbConfig, config *BackupConfig) { } } func startMultiBackup(bkConfig *BackupConfig, configFile string) { - utils.Info("Starting multiple backup jobs...") + utils.Info("Starting multiple backup job...") var conf = &Config{} conf, err := readConf(configFile) if err != nil { @@ -148,10 +150,6 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { } } -func intro() { - utils.Info("Starting PostgreSQL Backup...") - utils.Info("Copyright (c) 2024 Jonas Kaninda ") -} // BackupDatabase backup database func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) { @@ -182,7 +180,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool log.Fatal(err) } // save output - file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + file, err := os.Create(filepath.Join(tmpPath, backupFileName)) if err != nil { log.Fatal(err) } @@ -208,7 +206,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool gzipCmd := exec.Command("gzip") gzipCmd.Stdin = stdout // save output - gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName)) gzipCmd.Start() if err != nil { log.Fatal(err) @@ -243,6 +241,7 @@ func localBackup(db *dbConfig, config *BackupConfig) { } //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func s3Backup(db *dbConfig, config *BackupConfig) { @@ -283,6 +282,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") @@ -318,6 +319,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") @@ -353,6 +356,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func encryptBackup(config *BackupConfig) { diff --git a/pkg/config.go b/pkg/config.go index 8cd457c..5015fce 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -7,7 +7,6 @@ package pkg import ( - "errors" "fmt" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" @@ -271,5 +270,5 @@ func loadConfigFile() (string, error) { if err == nil { return backupConfigFile, nil } - return "", errors.New("backup config file not found") + return "", fmt.Errorf("backup config file not found") } diff --git a/pkg/helper.go b/pkg/helper.go index deb98f7..8586631 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -17,6 +17,11 @@ import ( "time" ) +func intro() { + utils.Info("Starting PostgreSQL Backup...") + utils.Info("Copyright (c) 2024 Jonas Kaninda ") +} + // copyToTmp copy file to temporary directory func copyToTmp(sourcePath string, backupFileName string) { //Copy backup from storage to /tmp @@ -186,8 +191,9 @@ func checkPrKeyFile(prKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } + +// readConf reads config file and returns Config func readConf(configFile string) (*Config, error) { - //configFile := filepath.Join("./", filename) if utils.FileExists(configFile) { buf, err := os.ReadFile(configFile) if err != nil { @@ -204,6 +210,8 @@ func readConf(configFile string) (*Config, error) { } return nil, fmt.Errorf("config file %q not found", configFile) } + +// checkConfigFile checks config files and returns one config file func checkConfigFile(filePath string) (string, error) { // Define possible config file names configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} diff --git a/utils/utils.go b/utils/utils.go index 8ecb140..15392b8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -20,6 +20,7 @@ import ( "strconv" ) +// FileExists checks if the file does exist func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) {