Merge pull request #1197 from neicnordic/feature/ingest-multiple-keys

Ingest supports multiple C4GH keys

.github/integration/scripts/make_sda_credentials.sh

@@ -61,10 +61,17 @@ if [ ! -f "/shared/crypt4gh" ]; then
     latest_c4gh=$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')
     curl --retry 100 -s -L "https://github.com/neicnordic/crypt4gh/releases/download/$latest_c4gh/crypt4gh_linux_x86_64.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/crypt4gh
 fi
+
 if [ ! -f "/shared/c4gh.sec.pem" ]; then
     echo "creating crypth4gh key"
     /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass
 fi
+
+if [ ! -f "/shared/c4gh1.sec.pem" ]; then
+    echo "creating crypth4gh key"
+    /shared/crypt4gh generate -n /shared/c4gh1 -p c4ghpass
+fi
+
 if [ ! -f "/shared/sync.sec.pem" ]; then
     echo "creating sync crypth4gh key"
     /shared/crypt4gh generate -n /shared/sync -p syncPass

.github/integration/sda/config.yaml

@@ -70,6 +70,11 @@ c4gh:
   filePath: /shared/c4gh.sec.pem
   passphrase: "c4ghpass"
   syncPubKeyPath: /shared/sync.pub.pem
+  privateKeys:
+  - filePath: /shared/c4gh.sec.pem
+    passphrase: "c4ghpass"
+  - filePath: /shared/c4gh1.sec.pem
+    passphrase: "c4ghpass"
 
 oidc:
   id:

.github/integration/tests/sda/11_api_test.sh

@@ -52,6 +52,12 @@ if [ "$resp" != "200" ]; then
 	exit 1
 fi
 
+ts=$(date +"%F %T")
+depr="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq -r .[1].deprecated_at)"
+if [ "$depr" != "$ts" ]; then
+	echo "Error when listing key hash, expected $ts got: $depr"
+	exit 1
+fi
 
 # list key hashes
 resp="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq '. | length')"
@@ -66,11 +72,5 @@ if [ "$resp" != "$manual_hash" ]; then
 	echo "Error when listing key hash, expected $manual_hash got: $resp"
 	exit 1
 fi
-ts=$(date +"%F %T")
-depr="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq -r .[1].deprecated_at)"
-if [ "$depr" != "$ts" ]; then
-	echo "Error when listing key hash, expected $ts got: $depr"
-	exit 1
-fi
 
 echo "api test completed successfully"

sda/cmd/ingest/ingest.go

@@ -62,7 +62,7 @@ func main() {
 		sigc <- syscall.SIGINT
 		panic(err)
 	}
-	key, err := config.GetC4GHKey()
+	archiveKeyList, err := config.GetC4GHprivateKeys()
 	if err != nil {
 		log.Error(err)
 		sigc <- syscall.SIGINT
@@ -384,11 +384,25 @@ func main() {
 
 					//nolint:nestif
 					if bytesRead <= int64(len(readBuffer)) {
-						header, err := tryDecrypt(key, readBuffer)
-						if err != nil {
-							log.Errorf("Trying to decrypt start of file failed, reason: (%s)", err.Error())
-							if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", fmt.Sprintf("{\"error\" : \"%s\"}", err.Error()), string(delivered.Body)); err != nil {
-								log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId)
+						var privateKey *[32]byte
+						var header []byte
+
+						// Iterate over the key list to try decryption
+						for _, key := range archiveKeyList {
+							header, err = tryDecrypt(key, readBuffer)
+							if err == nil {
+								privateKey = key
+
+								break
+							}
+							log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error())
+						}
+
+						// Check if decryption was successful with any key
+						if privateKey == nil {
+							log.Errorf("All keys failed to decrypt the submitted file")
+							if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(delivered.Body)); err != nil {
+								log.Errorf("Failed to set ingestion status for file from message: %v", delivered.CorrelationId)
 							}
 
 							if err := delivered.Ack(false); err != nil {
@@ -397,8 +411,8 @@ func main() {
 
 							// Send the message to an error queue so it can be analyzed.
 							fileError := broker.InfoError{
-								Error:           "Trying to decrypt start of file failed",
-								Reason:          err.Error(),
+								Error:           "Trying to decrypt the submitted file failed",
+								Reason:          "Decryption failed with the available key(s)",
 								OriginalMessage: message,
 							}
 							body, _ := json.Marshal(fileError)
@@ -409,8 +423,9 @@ func main() {
 							continue mainWorkLoop
 						}
 
+						// Proceed with the successful key
 						// Set the file's hex encoded public key
-						publicKey := keys.DerivePublicKey(*key)
+						publicKey := keys.DerivePublicKey(*privateKey)
 						keyhash := hex.EncodeToString(publicKey[:])
 						err = db.SetKeyHash(keyhash, fileID)
 						if err != nil {

sda/cmd/ingest/ingest_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/neicnordic/crypt4gh/keys"
 	"github.com/neicnordic/crypt4gh/streaming"
 	"github.com/neicnordic/sensitive-data-archive/internal/config"
+	"github.com/neicnordic/sensitive-data-archive/internal/helper"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
@@ -30,18 +31,22 @@ func (suite *TestSuite) SetupTest() {
 	defer os.RemoveAll(archive)
 	viper.Set("archive.location", archive)
 
-	// Generate a crypth4gh keypair
-	publicKey, privateKey, err := keys.GenerateKeyPair()
+	tempDir := suite.T().TempDir()
+	keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir)
+	keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir)
+
+	publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test")
 	assert.NoError(suite.T(), err)
+	// Add only the first public key to the list
 	pubKeyList = append(pubKeyList, publicKey)
 
-	tempDir := suite.T().TempDir()
-	privateKeyFile, err := os.Create(fmt.Sprintf("%s/c4fg.key", tempDir))
+	_, err = helper.CreatePrivateKeyFile(keyFile2, "test")
 	assert.NoError(suite.T(), err)
-	err = keys.WriteCrypt4GHX25519PrivateKey(privateKeyFile, privateKey, []byte("password"))
-	assert.NoError(suite.T(), err)
-	viper.Set("c4gh.filepath", fmt.Sprintf("%s/c4fg.key", tempDir))
-	viper.Set("c4gh.passphrase", "password")
+
+	viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{
+		{FilePath: keyFile1, Passphrase: "test"},
+		{FilePath: keyFile2, Passphrase: "test"},
+	})
 
 	viper.Set("broker.host", "test")
 	viper.Set("broker.port", 123)
@@ -67,10 +72,12 @@ func (suite *TestSuite) TestTryDecrypt_wrongFile() {
 	buf, err := io.ReadAll(file)
 	assert.NoError(suite.T(), err)
 
-	key, err := config.GetC4GHKey()
-	assert.Nil(suite.T(), err)
-	b, err := tryDecrypt(key, buf)
-	assert.Nil(suite.T(), b)
+	privateKeys, err := config.GetC4GHprivateKeys()
+	assert.NoError(suite.T(), err)
+	assert.Len(suite.T(), privateKeys, 2)
+
+	header, err := tryDecrypt(privateKeys[0], buf)
+	assert.Nil(suite.T(), header)
 	assert.EqualError(suite.T(), err, "not a Crypt4GH file")
 }
 
@@ -102,9 +109,18 @@ func (suite *TestSuite) TestTryDecrypt() {
 	buf, err := io.ReadAll(file)
 	assert.NoError(suite.T(), err)
 
-	key, err := config.GetC4GHKey()
+	privateKeys, err := config.GetC4GHprivateKeys()
 	assert.NoError(suite.T(), err)
-	header, err := tryDecrypt(key, buf)
-	assert.NoError(suite.T(), err)
-	assert.NotNil(suite.T(), header)
+
+	for i, key := range privateKeys {
+		header, err := tryDecrypt(key, buf)
+		switch {
+		case i == 0:
+			assert.NoError(suite.T(), err)
+			assert.NotNil(suite.T(), header)
+		default:
+			assert.Contains(suite.T(), err.Error(), "could not find matching public key heade")
+			assert.Nil(suite.T(), header)
+		}
+	}
 }

sda/config_local.yaml

@@ -72,6 +72,11 @@ c4gh:
   filePath: "/tmp/shared/c4gh.sec.pem"
   passphrase: "c4ghpass"
   syncPubKeyPath: "/tmp/shared/sync.pub.pem"
+  privateKeys:
+    - filePath: "/tmp/shared/c4gh.sec.pem"
+      passphrase: "c4ghpass"
+    - filePath: "/tmp/shared/c4gh1.sec.pem"
+      passphrase: "c4ghpass"
 
 oidc:
   configuration:

sda/internal/config/config.go

@@ -151,6 +151,11 @@ type CORSConfig struct {
 	AllowCredentials bool
 }
 
+type C4GHprivateKeyConf struct {
+	FilePath   string `mapstructure:"filePath"`
+	Passphrase string `mapstructure:"passphrase"`
+}
+
 // NewConfig initializes and parses the config file and/or environment using
 // the viper library.
 func NewConfig(app string) (*Config, error) {
@@ -1066,6 +1071,34 @@ func GetC4GHKey() (*[32]byte, error) {
 	return &key, nil
 }
 
+// GetC4GHprivateKeys reads and decrypts keys and returns a list of c4gh keys
+func GetC4GHprivateKeys() ([]*[32]byte, error) {
+	// Retrieve the list of key configurations from the YAML file
+	var keySet []C4GHprivateKeyConf
+	if err := viper.UnmarshalKey("c4gh.privateKeys", &keySet); err != nil {
+		return nil, fmt.Errorf("failed to parse key configurations: %v", err)
+	}
+
+	var privateKeys []*[32]byte
+
+	for _, entry := range keySet {
+		keyFile, err := os.Open(entry.FilePath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open key file %s: %v", entry.FilePath, err)
+		}
+
+		key, err := keys.ReadPrivateKey(keyFile, []byte(entry.Passphrase))
+		keyFile.Close()
+		if err != nil {
+			return nil, fmt.Errorf("failed to read private key from %s: %v", entry.FilePath, err)
+		}
+
+		privateKeys = append(privateKeys, &key)
+	}
+
+	return privateKeys, nil
+}
+
 // GetC4GHPublicKey reads the c4gh public key
 func GetC4GHPublicKey() (*[32]byte, error) {
 	keyPath := viper.GetString("c4gh.syncPubKeyPath")

sda/internal/config/config_test.go

@@ -11,10 +11,8 @@ import (
 	"testing"
 	"time"
 
-	helper "github.com/neicnordic/sensitive-data-archive/internal/helper"
-
+	"github.com/neicnordic/sensitive-data-archive/internal/helper"
 	log "github.com/sirupsen/logrus"
-
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
@@ -364,6 +362,78 @@ func (suite *ConfigTestSuite) TestGetC4GHKey() {
 	defer os.RemoveAll(keyPath)
 }
 
+func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() {
+	keyPath, _ := os.MkdirTemp("", "key")
+	keyFile1 := keyPath + "/c4gh1.key"
+	keyFile2 := keyPath + "/c4gh2.key"
+
+	_, err := helper.CreatePrivateKeyFile(keyFile1, "test")
+	assert.NoError(suite.T(), err)
+	_, err = helper.CreatePrivateKeyFile(keyFile2, "test")
+	assert.NoError(suite.T(), err)
+
+	viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{
+		{FilePath: keyFile1, Passphrase: "test"},
+		{FilePath: keyFile2, Passphrase: "test"},
+	})
+
+	privateKeys, err := GetC4GHprivateKeys()
+	assert.NoError(suite.T(), err)
+	assert.Len(suite.T(), privateKeys, 2)
+
+	defer os.RemoveAll(keyPath)
+}
+
+func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_MissingKeyPath() {
+	viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{
+		{FilePath: "/non/existent/path1", Passphrase: "test"},
+	})
+
+	privateKeys, err := GetC4GHprivateKeys()
+	assert.Error(suite.T(), err)
+	assert.Contains(suite.T(), err.Error(), "failed to open key file")
+	assert.Nil(suite.T(), privateKeys)
+}
+
+func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_WrongPassphrase() {
+	keyPath, _ := os.MkdirTemp("", "key")
+	keyFile := keyPath + "/c4gh1.key"
+
+	_, err := helper.CreatePrivateKeyFile(keyFile, "test")
+	assert.NoError(suite.T(), err)
+
+	viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{
+		{FilePath: keyFile, Passphrase: "wrong"},
+	})
+
+	privateKeys, err := GetC4GHprivateKeys()
+	assert.Error(suite.T(), err)
+	assert.Contains(suite.T(), err.Error(), "chacha20poly1305: message authentication faile")
+	assert.Nil(suite.T(), privateKeys)
+
+	defer os.RemoveAll(keyPath)
+}
+
+func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_InvalidKey() {
+	key := "not a valid key"
+	keyPath, _ := os.MkdirTemp("", "key")
+	keyFile := keyPath + "/c4gh1.key"
+
+	err := os.WriteFile(keyFile, []byte(key), 0600)
+	assert.NoError(suite.T(), err)
+
+	viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{
+		{FilePath: keyFile, Passphrase: "wrong"},
+	})
+
+	privateKeys, err := GetC4GHprivateKeys()
+	assert.Error(suite.T(), err)
+	assert.Contains(suite.T(), err.Error(), "read of unrecognized private key format")
+	assert.Nil(suite.T(), privateKeys)
+
+	defer os.RemoveAll(keyPath)
+}
+
 func (suite *ConfigTestSuite) TestConfigSyncAPI() {
 	suite.SetupTest()
 	noConfig, err := NewConfig("sync-api")

sda/internal/helper/helper.go

@@ -20,6 +20,7 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwa"
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/lestrrat-go/jwx/v2/jwt"
+	"github.com/neicnordic/crypt4gh/keys"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -421,3 +422,27 @@ func TLScertToFile(filename string, derBytes []byte) error {
 
 	return err
 }
+
+// CreatePrivateKeyFile creates a private key file with the given passphrase
+// and returns the generated public key.
+func CreatePrivateKeyFile(keyFile string, passphrase string) ([32]byte, error) {
+	var zeroPubKey [32]byte
+
+	publicKey, privateKey, err := keys.GenerateKeyPair()
+	if err != nil {
+		return zeroPubKey, err
+	}
+
+	keyFileWriter, err := os.Create(keyFile)
+	if err != nil {
+		return zeroPubKey, err
+	}
+	defer keyFileWriter.Close()
+
+	// Write the private key to the file
+	if err := keys.WriteCrypt4GHX25519PrivateKey(keyFileWriter, privateKey, []byte(passphrase)); err != nil {
+		return zeroPubKey, err
+	}
+
+	return publicKey, nil
+}