Support Direct Import of Certificates via URL Feature req #3350

Adding the import-cert-url command to import a cert with a url

agent/provider/Cargo.toml

@@ -66,9 +66,11 @@ sys-info = "0.8.0"
 thiserror = "1.0.14"
 tokio = { version = "1", features = ["macros", "process", "signal"] }
 tokio-stream = { version = "0.1.6", features = ["sync"] }
-url = "2.1.1"
+url = "2.5"
 walkdir = "2.3.1"
 yansi = "0.5.0"
+reqwest = { version = "0.11", features = ["blocking"] }
+tempfile = "3.8"
 
 [target.'cfg(target_family = "unix")'.dependencies]
 nix = "0.22.0"

agent/provider/src/cert_utils.rs

@@ -0,0 +1,44 @@
+use anyhow::{anyhow, Result};
+use std::path::{Path, PathBuf};
+use url::Url;
+use std::fs;
+
+pub fn download_cert_if_url(cert_source: &PathBuf, temp_dir: &Path) -> Result<PathBuf> {
+    let source_str = cert_source.to_string_lossy();
+    if let Ok(url) = Url::parse(&source_str) {
+        if url.scheme() == "http" || url.scheme() == "https" {
+            download_cert(&source_str, temp_dir)
+        } else {
+            Ok(cert_source.clone())
+        }
+    } else {
+        Ok(cert_source.clone())
+    }
+}
+
+fn download_cert(url: &str, temp_dir: &Path) -> Result<PathBuf> {
+    // Create temp directory if it doesn't exist
+    fs::create_dir_all(temp_dir)?;
+
+    // Generate a unique filename based on the URL
+    let file_name = url
+        .split('/')
+        .last()
+        .ok_or_else(|| anyhow!("Invalid URL format"))?;
+    let temp_path = temp_dir.join(file_name);
+
+    // Download the certificate
+    let response = reqwest::blocking::get(url)?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "Failed to download certificate. Status: {}",
+            response.status()
+        ));
+    }
+
+    // Save the certificate to a temporary file
+    let content = response.bytes()?;
+    fs::write(&temp_path, content)?;
+
+    Ok(temp_path)
+}

agent/provider/src/cli/rule.rs

@@ -103,6 +103,13 @@ pub enum AuditedPayloadRuleWithCert {
         #[structopt(short, long, possible_values = Mode::VARIANTS)]
         mode: Mode,
     },
+    /// Import and set rule for X509 certificate from URL.
+    ImportCertUrl {
+        /// URL to X509 certificate or X509 certificates chain.
+        url: String,
+        #[structopt(short, long, possible_values = Mode::VARIANTS)]
+        mode: Mode,
+    },
 }
 
 #[derive(StructOpt, Clone, Debug)]
@@ -116,6 +123,13 @@ pub enum PartnerRuleWithCert {
         #[structopt(short, long, possible_values = Mode::VARIANTS)]
         mode: Mode,
     },
+    /// Import and set rule for Golem certificate from URL.
+    ImportCertUrl {
+        /// URL to Golem certificate.
+        url: String,
+        #[structopt(short, long, possible_values = Mode::VARIANTS)]
+        mode: Mode,
+    },
 }
 
 #[derive(StructOpt, Clone, Debug)]
@@ -271,6 +285,36 @@ fn set(set_rule: SetRule, config: ProviderConfig) -> Result<()> {
 
                 Ok(())
             }
+            SetOutboundRule::AuditedPayload(AuditedPayloadRuleWithCert::ImportCertUrl {
+                url,
+                mode,
+            }) => {
+                // TODO change it to `rules.keystore.add` when AuditedPayload will support Golem certs.
+                let AddResponse {
+                    invalid,
+                    leaf_cert_ids,
+                    duplicated,
+                    ..
+                } = rules.keystore.add_x509_cert(&AddParams {
+                    certs: vec![PathBuf::from(url)],
+                })?;
+
+                for cert_path in invalid {
+                    log::error!("Failed to import X509 certificates from: {cert_path:?}.");
+                }
+
+                rules.keystore.reload()?;
+
+                if leaf_cert_ids.is_empty() && !duplicated.is_empty() {
+                    log::warn!("Certificate is already in keystore- please use `cert-id` instead of `import-cert`");
+                }
+
+                for cert_id in leaf_cert_ids {
+                    rules.set_audited_payload_mode(cert_id, mode.clone())?;
+                }
+
+                Ok(())
+            }
             SetOutboundRule::Partner(PartnerRuleWithCert::CertId(CertId { cert_id, mode })) => {
                 rules.set_partner_mode(cert_id, mode)
             }
@@ -283,6 +327,17 @@ fn set(set_rule: SetRule, config: ProviderConfig) -> Result<()> {
                     rules.set_partner_mode(cert_id, mode.clone())?;
                 }
 
+                Ok(())
+            }
+            SetOutboundRule::Partner(PartnerRuleWithCert::ImportCertUrl {
+                url,
+                mode,
+            }) => {
+                let leaf_cert_ids = rules.import_certs_from_url(&url)?;
+                for cert_id in leaf_cert_ids {
+                    rules.set_partner_mode(cert_id, mode.clone())?;
+                }
+
                 Ok(())
             }
         },

agent/provider/src/lib.rs

@@ -13,6 +13,7 @@ pub mod rules;
 pub mod signal;
 pub mod startup_config;
 pub mod tasks;
+mod cert_utils;
 
 pub use config::globals::GlobalsState;
 pub use startup_config::ReceiverAccount;

agent/provider/src/rules.rs

@@ -6,6 +6,7 @@ use crate::rules::outbound::{CertRule, Mode, OutboundRules};
 use crate::rules::restrict::{AllowOnly, Blacklist, RestrictRule, RuleAccessor};
 use crate::rules::store::Rulestore;
 use crate::startup_config::FileMonitor;
+use crate::cert_utils::download_cert_if_url;
 
 use anyhow::{bail, Result};
 use golem_certificate::schemas::certificate::Fingerprint;
@@ -16,6 +17,7 @@ use std::{
     path::{Path, PathBuf},
 };
 use strum_macros::Display;
+use tempfile;
 
 use ya_client_model::NodeId;
 use ya_manifest_utils::keystore::{AddParams, AddResponse};
@@ -31,6 +33,7 @@ pub struct RulesManager {
     pub keystore: CompositeKeystore,
     whitelist: DomainWhitelistState,
     whitelist_file: PathBuf,
+    temp_dir: PathBuf,
 }
 
 impl RulesManager {
@@ -46,11 +49,17 @@ impl RulesManager {
 
         let rulestore = Rulestore::load_or_create(rules_file)?;
 
+        let temp_dir = tempfile::Builder::new()
+            .prefix("ya-provider-certs-")
+            .tempdir()?
+            .into_path();
+
         let manager = Self {
             whitelist_file: whitelist_file.to_path_buf(),
             rulestore,
             keystore,
             whitelist,
+            temp_dir,
         };
 
         manager.remove_dangling_rules()?;
@@ -115,6 +124,12 @@ impl RulesManager {
         Ok(leaf_cert_ids)
     }
 
+    /// Import certificates from a URL
+    pub fn import_certs_from_url(&mut self, url: &str) -> Result<Vec<Fingerprint>> {
+        let cert_path = download_cert_if_url(&PathBuf::from(url), &self.temp_dir)?;
+        self.import_certs(&cert_path)
+    }
+
     pub fn set_audited_payload_mode(&self, cert_id: String, mode: Mode) -> Result<()> {
         let cert_id = {
             let certs: Vec<Cert> = self