From 051135552f1c426d90e2e1f7ac5a4e978f894d76 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Tue, 21 Jan 2025 09:45:58 +0100 Subject: [PATCH] rofl-containers: Add support for container secret provisioning --- Cargo.lock | 3 +- rofl-appd/Cargo.toml | 1 + rofl-appd/src/lib.rs | 1 + rofl-appd/src/services/kms.rs | 136 ++++++++++++++++-- rofl-appd/src/types.rs | 15 ++ rofl-containers/Cargo.toml | 2 +- rofl-containers/src/containers.rs | 52 +++++++ rofl-containers/src/main.rs | 31 +++- rofl-containers/src/secrets.rs | 43 ++++++ runtime-sdk/src/modules/rofl/app/env.rs | 10 +- runtime-sdk/src/modules/rofl/app/mod.rs | 5 +- runtime-sdk/src/modules/rofl/app/processor.rs | 9 +- 12 files changed, 280 insertions(+), 28 deletions(-) create mode 100644 rofl-appd/src/types.rs create mode 100644 rofl-containers/src/containers.rs create mode 100644 rofl-containers/src/secrets.rs diff --git a/Cargo.lock b/Cargo.lock index 420557f80b..9e0b157b0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3960,6 +3960,7 @@ name = "rofl-appd" version = "0.1.0" dependencies = [ "anyhow", + "oasis-cbor", "oasis-runtime-sdk", "rocket", "rustc-hex", @@ -3973,7 +3974,7 @@ dependencies = [ [[package]] name = "rofl-containers" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "base64", diff --git a/rofl-appd/Cargo.toml b/rofl-appd/Cargo.toml index 9a1c8c6d72..08624f1175 100644 --- a/rofl-appd/Cargo.toml +++ b/rofl-appd/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] # Oasis SDK. +cbor = { version = "0.5.1", package = "oasis-cbor" } oasis-runtime-sdk = { path = "../runtime-sdk", features = ["tdx"] } # Third party. diff --git a/rofl-appd/src/lib.rs b/rofl-appd/src/lib.rs index 7faa846b94..a15137299c 100644 --- a/rofl-appd/src/lib.rs +++ b/rofl-appd/src/lib.rs @@ -3,6 +3,7 @@ mod routes; pub mod services; pub(crate) mod state; +pub mod types; use std::sync::Arc; diff --git a/rofl-appd/src/services/kms.rs b/rofl-appd/src/services/kms.rs index ae4d3e851a..721a38725d 100644 --- a/rofl-appd/src/services/kms.rs +++ b/rofl-appd/src/services/kms.rs @@ -7,11 +7,17 @@ use sp800_185::KMac; use tokio::sync::Notify; use oasis_runtime_sdk::{ - core::common::logger::get_logger, + core::common::{ + crypto::{mrae::deoxysii, x25519}, + logger::get_logger, + }, crypto::signature::{ed25519, secp256k1, Signer}, + modules, modules::rofl::app::{client::DeriveKeyRequest, prelude::*}, }; +use crate::types::SecretEnvelope; + /// A key management service. #[async_trait] pub trait KmsService: Send + Sync { @@ -23,14 +29,26 @@ pub trait KmsService: Send + Sync { /// Generate a key based on the passed parameters. async fn generate(&self, request: &GenerateRequest<'_>) -> Result; + + /// Decrypt and authenticate a secret using the secret encryption key (SEK). + async fn open_secret( + &self, + request: &OpenSecretRequest<'_>, + ) -> Result; } /// Error returned by the key management service. #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("invalid argument")] + InvalidArgument, + #[error("not initialized yet")] NotInitialized, + #[error("corrupted secret")] + CorruptedSecret, + #[error("internal error")] Internal, @@ -85,6 +103,26 @@ pub struct GenerateResponse { pub key: Vec, } +/// Secret decryption and authentication request. +#[derive(Clone, Debug, Default, serde::Deserialize)] +pub struct OpenSecretRequest<'r> { + /// Plain-text name associated with the secret. + pub name: &'r str, + /// Encrypted secret value. + /// + /// It is expected that the value contains a CBOR-encoded `SecretEnvelope`. + pub value: &'r [u8], +} + +/// Secret decryption and authentication response. +#[derive(Clone, Default, serde::Serialize)] +pub struct OpenSecretResponse { + /// Decrypted plain-text name. + pub name: Vec, + /// Decrypted plain-text value. + pub value: Vec, +} + /// Key identifier for the root key from which all per-app keys are derived. The root key is /// retrieved from the Oasis runtime key manager on initialization and all subsequent keys are /// derived from that key. @@ -92,23 +130,28 @@ pub struct GenerateResponse { /// Changing this identifier will change all generated keys. const OASIS_KMS_ROOT_KEY_ID: &[u8] = b"oasis-runtime-sdk/rofl-appd: root key v1"; +struct Keys { + root: Vec, + sek: x25519::PrivateKey, +} + /// A key management service backed by the Oasis runtime. pub struct OasisKmsService { running: AtomicBool, - root_key: Arc>>>, env: Environment, logger: slog::Logger, ready_notify: Notify, + keys: Arc>>, } impl OasisKmsService { pub fn new(env: Environment) -> Self { Self { running: AtomicBool::new(false), - root_key: Arc::new(Mutex::new(None)), env, logger: get_logger("appd/services/kms"), ready_notify: Notify::new(), + keys: Arc::new(Mutex::new(None)), } } } @@ -127,9 +170,11 @@ impl KmsService for OasisKmsService { slog::info!(self.logger, "starting KMS service"); // Ensure we keep retrying until the root key is derived. - let retry_strategy = tokio_retry::strategy::ExponentialBackoff::from_millis(4) - .max_delay(std::time::Duration::from_millis(1000)) - .map(tokio_retry::strategy::jitter); + let retry_strategy = || { + tokio_retry::strategy::ExponentialBackoff::from_millis(4) + .max_delay(std::time::Duration::from_millis(1000)) + .map(tokio_retry::strategy::jitter) + }; slog::info!( self.logger, @@ -138,19 +183,41 @@ impl KmsService for OasisKmsService { // Generate the root key for the application and store it in memory to derive all other // requested keys. - let root_key = tokio_retry::Retry::spawn(retry_strategy, || { + let root_key_task = tokio_retry::Retry::spawn(retry_strategy(), || { self.env.client().derive_key( self.env.signer(), DeriveKeyRequest { key_id: OASIS_KMS_ROOT_KEY_ID.to_vec(), + kind: modules::rofl::types::KeyKind::EntropyV0, ..Default::default() }, ) - }) - .await?; + }); - // Store the key in memory. - *self.root_key.lock().unwrap() = Some(root_key.key); + // Generate the secrets encryption key (SEK) and store it in memory. + // TODO: Consider caching key in encrypted persistent storage. + let sek_task = tokio_retry::Retry::spawn(retry_strategy(), || { + self.env.client().derive_key( + self.env.identity(), + DeriveKeyRequest { + key_id: modules::rofl::ROFL_KEY_ID_SEK.to_vec(), + kind: modules::rofl::types::KeyKind::X25519, + ..Default::default() + }, + ) + }); + + // Perform requests in parallel. + let (root_key, sek) = tokio::try_join!(root_key_task, sek_task,)?; + + let sek: [u8; 32] = sek.key.try_into().map_err(|_| Error::Internal)?; + let sek = sek.into(); + + // Store the keys in memory. + *self.keys.lock().unwrap() = Some(Keys { + root: root_key.key, + sek, + }); self.ready_notify.notify_waiters(); @@ -162,7 +229,7 @@ impl KmsService for OasisKmsService { async fn wait_ready(&self) -> Result<(), Error> { let handle = self.ready_notify.notified(); - if self.root_key.lock().unwrap().is_some() { + if self.keys.lock().unwrap().is_some() { return Ok(()); } @@ -172,13 +239,47 @@ impl KmsService for OasisKmsService { } async fn generate(&self, request: &GenerateRequest<'_>) -> Result { - let root_key_guard = self.root_key.lock().unwrap(); - let root_key = root_key_guard.as_ref().ok_or(Error::NotInitialized)?; + let keys_guard = self.keys.lock().unwrap(); + let root_key = &keys_guard.as_ref().ok_or(Error::NotInitialized)?.root; let key = Kdf::derive_key(root_key.as_ref(), request.kind, request.key_id.as_bytes())?; Ok(GenerateResponse { key }) } + + async fn open_secret( + &self, + request: &OpenSecretRequest<'_>, + ) -> Result { + let envelope: SecretEnvelope = + cbor::from_slice(request.value).map_err(|_| Error::InvalidArgument)?; + + let keys_guard = self.keys.lock().unwrap(); + let sek = &keys_guard.as_ref().ok_or(Error::NotInitialized)?.sek; + let sek = sek.clone().into(); // Fine as the clone will be zeroized on drop. + + // Name. + let name = deoxysii::box_open( + &envelope.nonce, + envelope.name.clone(), + b"name".into(), // Prevent mixing name and value. + &envelope.pk.0, + &sek, + ) + .map_err(|_| Error::CorruptedSecret)?; + + // Value. + let value = deoxysii::box_open( + &envelope.nonce, + envelope.value.clone(), + b"value".into(), // Prevent mixing name and value. + &envelope.pk.0, + &sek, + ) + .map_err(|_| Error::CorruptedSecret)?; + + Ok(OpenSecretResponse { name, value }) + } } /// Insecure mock root key used to derive keys in the mock KMS. @@ -206,6 +307,13 @@ impl KmsService for MockKmsService { Ok(GenerateResponse { key }) } + + async fn open_secret( + &self, + _request: &OpenSecretRequest<'_>, + ) -> Result { + Err(Error::NotInitialized) + } } /// Domain separation tag for deriving a key-derivation key from a shared secret. diff --git a/rofl-appd/src/types.rs b/rofl-appd/src/types.rs new file mode 100644 index 0000000000..5596ef0e03 --- /dev/null +++ b/rofl-appd/src/types.rs @@ -0,0 +1,15 @@ +//! Various types used by rofl-appd. +use oasis_runtime_sdk::core::common::crypto::{mrae::deoxysii, x25519}; + +/// Envelope used for storing encrypted secrets. +#[derive(Clone, Debug, Default, cbor::Encode, cbor::Decode)] +pub struct SecretEnvelope { + /// Ephemeral public key used for X25519. + pub pk: x25519::PublicKey, + /// Nonce. + pub nonce: [u8; deoxysii::NONCE_SIZE], + /// Encrypted secret name. + pub name: Vec, + /// Encrypted secret value. + pub value: Vec, +} diff --git a/rofl-containers/Cargo.toml b/rofl-containers/Cargo.toml index 173780a175..58a23b3ef0 100644 --- a/rofl-containers/Cargo.toml +++ b/rofl-containers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rofl-containers" -version = "0.2.1" +version = "0.3.0" edition = "2021" [dependencies] diff --git a/rofl-containers/src/containers.rs b/rofl-containers/src/containers.rs new file mode 100644 index 0000000000..664ccf06ec --- /dev/null +++ b/rofl-containers/src/containers.rs @@ -0,0 +1,52 @@ +use anyhow::Result; +use cmd_lib::run_cmd; + +/// Initialize container environment. +pub async fn init() -> Result<()> { + // Setup networking. + run_cmd!( + mount none -t tmpfs "/tmp"; + udhcpc -i eth0 -q -n; + )?; + + // Mount cgroups and create /dev/shm for Podman locks. + run_cmd!( + mount -t cgroup2 none "/sys/fs/cgroup"; + mkdir -p "/dev/shm"; + mount -t tmpfs none "/dev/shm"; + )?; + + // Cleanup state after reboot. + run_cmd!( + rm -rf "/storage/containers/run"; + rm -rf "/storage/containers/net"; + rm -rf "/var/lib/cni"; + + mkdir -p "/storage/containers/run"; + mkdir -p "/storage/containers/graph"; + mkdir -p "/storage/containers/graph/tmp"; + mkdir -p "/storage/containers/net"; + )?; + + // Update TUN device permissions. + run_cmd!(chmod 0666 "/dev/net/tun")?; + + // Migrate existing containers if needed. + run_cmd!( + podman system migrate; + podman system prune --external; + )?; + + Ok(()) +} + +/// Start containers. +pub async fn start() -> Result<()> { + // Bring containers up. + run_cmd!( + cd "/etc/oasis/containers"; + podman-compose up --detach --remove-orphans --force-recreate; + )?; + + Ok(()) +} diff --git a/rofl-containers/src/main.rs b/rofl-containers/src/main.rs index e3552ba022..25923c2a71 100644 --- a/rofl-containers/src/main.rs +++ b/rofl-containers/src/main.rs @@ -18,7 +18,9 @@ use oasis_runtime_sdk::{ }; use rofl_appd::services; +mod containers; mod reaper; +mod secrets; mod storage; /// UNIX socket address where the REST API server will listen on. @@ -63,17 +65,42 @@ impl App for ContainersApp { let _ = kms.wait_ready().await; // Initialize storage when configured in the kernel cmdline. + slog::info!(logger, "initializing stage 2 storage"); if let Err(err) = storage::init(kms.clone()).await { slog::error!(logger, "failed to initialize stage 2 storage"; "err" => ?err); process::abort(); } // Start the REST API server. + slog::info!(logger, "starting the API server"); let cfg = rofl_appd::Config { address: ROFL_APPD_ADDRESS, - kms, + kms: kms.clone(), }; - let _ = rofl_appd::start(cfg, env).await; + let _ = rofl_appd::start(cfg, env.clone()).await; + + // Initialize containers. + slog::info!(logger, "initializing container environment"); + if let Err(err) = containers::init().await { + slog::error!(logger, "failed to initialize container environment"; "err" => ?err); + process::abort(); + } + + // Initialize secrets. + slog::info!(logger, "initializing container secrets"); + if let Err(err) = secrets::init(env.clone(), kms.clone()).await { + slog::error!(logger, "failed to initialize container secrets"; "err" => ?err); + process::abort(); + } + + // Start containers. + slog::info!(logger, "starting containers"); + if let Err(err) = containers::start().await { + slog::error!(logger, "failed to start containers"; "err" => ?err); + process::abort(); + } + + slog::info!(logger, "everything is up and running"); } } diff --git a/rofl-containers/src/secrets.rs b/rofl-containers/src/secrets.rs new file mode 100644 index 0000000000..15388c7322 --- /dev/null +++ b/rofl-containers/src/secrets.rs @@ -0,0 +1,43 @@ +use std::sync::Arc; + +use anyhow::Result; +use cmd_lib::run_cmd; + +use oasis_runtime_sdk::{core::common::logger::get_logger, modules::rofl::app::prelude::*}; +use rofl_appd::services::{self, kms::OpenSecretRequest}; + +/// Initialize secrets available to containers. +pub async fn init( + env: Environment, + kms: Arc, +) -> Result<()> { + let logger = get_logger("secrets"); + + // Query own app cfg to get encrypted secrets. + let encrypted_secrets = env.client().app_cfg().await?.secrets; + + // Ensure all secrets are removed. + run_cmd!(podman secret rm --all)?; + // Create all requested secrets. + for (pub_name, value) in encrypted_secrets { + // Decrypt and authenticate secret. In case of failures, the secret is skipped. + let (name, value) = match kms + .open_secret(&OpenSecretRequest { + name: &pub_name, + value: &value, + }) + .await + { + Ok(response) => (response.name, response.value), + Err(_) => continue, // Skip bad secrets. + }; + // Assume the name and value are always valid strings. + let name = String::from_utf8_lossy(&name); + let value = String::from_utf8_lossy(&value); + // Create a new Podman secret in temporary storage on /run to avoid it being persisted. + let _ = run_cmd!(echo -n $value | podman secret create --driver-opts file=/run/podman/secrets --replace $name -); + + slog::info!(logger, "provisioned secret"; "pub_name" => pub_name); + } + Ok(()) +} diff --git a/runtime-sdk/src/modules/rofl/app/env.rs b/runtime-sdk/src/modules/rofl/app/env.rs index c58ae5686f..a87672d4df 100644 --- a/runtime-sdk/src/modules/rofl/app/env.rs +++ b/runtime-sdk/src/modules/rofl/app/env.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use tokio::sync::mpsc; -use crate::crypto::signature::Signer; +use crate::{core::identity::Identity, crypto::signature::Signer}; use super::{client, processor, App}; @@ -12,6 +12,7 @@ pub struct Environment { app: Arc, client: client::Client, signer: Arc, + identity: Arc, cmdq: mpsc::WeakSender, } @@ -27,6 +28,7 @@ where Self { app: state.app.clone(), signer: state.signer.clone(), + identity: state.identity.clone(), client: client::Client::new(state, cmdq.clone()), cmdq, } @@ -47,6 +49,11 @@ where self.signer.clone() } + /// Runtime identity. + pub fn identity(&self) -> Arc { + self.identity.clone() + } + /// Send a command to the processor. pub(super) async fn send_command(&self, cmd: processor::Command) -> Result<()> { let cmdq = self @@ -66,6 +73,7 @@ where Self { app: self.app.clone(), signer: self.signer.clone(), + identity: self.identity.clone(), client: self.client.clone(), cmdq: self.cmdq.clone(), } diff --git a/runtime-sdk/src/modules/rofl/app/mod.rs b/runtime-sdk/src/modules/rofl/app/mod.rs index b3f114a2c3..ef7e7b05c6 100644 --- a/runtime-sdk/src/modules/rofl/app/mod.rs +++ b/runtime-sdk/src/modules/rofl/app/mod.rs @@ -21,7 +21,7 @@ use crate::{ pub mod client; mod env; -mod init; +pub mod init; mod notifier; pub mod prelude; mod processor; @@ -83,7 +83,8 @@ pub trait App: Send + Sync + 'static { where Self: Sized, { - // Default implementation does nothing. + // Default implementation just runs the trivial initialization. + init::post_registration_init(); } /// Main application processing loop. diff --git a/runtime-sdk/src/modules/rofl/app/processor.rs b/runtime-sdk/src/modules/rofl/app/processor.rs index e4936cf6ab..eaa3b73743 100644 --- a/runtime-sdk/src/modules/rofl/app/processor.rs +++ b/runtime-sdk/src/modules/rofl/app/processor.rs @@ -16,7 +16,7 @@ use crate::{ }; use rand::rngs::OsRng; -use super::{init, notifier, registration, App, Environment}; +use super::{notifier, registration, App, Environment}; /// Size of the processor command queue. const CMDQ_BACKLOG: usize = 32; @@ -189,13 +189,8 @@ where logger, "performing app-specific post-registration initialization" ); - app.post_registration_init(env).await; - slog::info!( - logger, - "performing additional post-registration initialization" - ); - init::post_registration_init(); + app.post_registration_init(env).await; }); // Notify notifier task.