From 2d100611c33f1384dfd3e5b51b7935f29912e536 Mon Sep 17 00:00:00 2001
From: Gianmarco Fraccaroli <gianmarco.fraccaroli@gmail.com>
Date: Fri, 22 Nov 2024 14:40:19 +0100
Subject: [PATCH 1/2] retry migrations

---
 chain/src/main.rs        | 41 ++++++++++++++++++++++++++++++++++------
 chain/src/services/db.rs |  2 +-
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/chain/src/main.rs b/chain/src/main.rs
index bdc9eb2..ded8007 100644
--- a/chain/src/main.rs
+++ b/chain/src/main.rs
@@ -4,8 +4,10 @@ pub mod entity;
 pub mod services;
 
 use std::collections::BTreeMap;
+use std::env;
 use std::sync::atomic::{self, AtomicBool};
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::Context;
 use clap::Parser;
@@ -16,6 +18,7 @@ use shared::transaction::Transaction;
 use shared::tx_index::{MaspTxIndex, TxIndex};
 use tendermint_rpc::HttpClient;
 use tokio::signal;
+use tokio::time::sleep;
 use tokio_retry::strategy::{jitter, FixedInterval};
 use tokio_retry::RetryIf;
 
@@ -106,12 +109,38 @@ fn must_exit_handle() -> Arc<AtomicBool> {
 }
 
 async fn run_migrations(app_state: &AppState) -> Result<(), MainError> {
-    db_service::run_migrations(
-        app_state.get_db_connection().await.into_db_error()?,
-    )
-    .await
-    .context("Failed to run db migrations")
-    .into_db_error()
+    let mut max_retries = env::var("DATABASE_MAX_MIGRATION_RETRY")
+        .unwrap_or_else(|_| 5.to_string())
+        .parse::<u64>()
+        .unwrap_or(5_u64);
+    loop {
+        let migration_res = db_service::run_migrations(
+            app_state.get_db_connection().await.into_db_error()?,
+        )
+        .await;
+
+        match &migration_res {
+            Ok(_) => {
+                return migration_res
+                    .context("Failed to run db migrations")
+                    .into_db_error()
+            }
+            Err(e) => {
+                tracing::debug!(
+                    "Failed runnign migrations: {} ({}/5)",
+                    e.to_string(),
+                    max_retries
+                );
+                if max_retries == 0 {
+                    return migration_res
+                        .context("Failed to run db migrations")
+                        .into_db_error();
+                }
+                max_retries -= 1;
+                sleep(Duration::from_secs(3)).await;
+            }
+        }
+    }
 }
 
 async fn load_committed_state(
diff --git a/chain/src/services/db.rs b/chain/src/services/db.rs
index c3e66ac..6b98b80 100644
--- a/chain/src/services/db.rs
+++ b/chain/src/services/db.rs
@@ -36,7 +36,7 @@ pub async fn run_migrations(conn: Object) -> anyhow::Result<()> {
     conn.interact(|transaction_conn| {
         transaction_conn
             .run_pending_migrations(MIGRATIONS)
-            .map_err(|_| anyhow!("Failed to run db migrations"))?;
+            .map_err(|e| anyhow!("Failed to run db migrations: {}", e.to_string()))?;
         anyhow::Ok(())
     })
     .await

From d5e1739cce03f7c08684211b1dc874464dd42cb9 Mon Sep 17 00:00:00 2001
From: Gianmarco Fraccaroli <gianmarco.fraccaroli@gmail.com>
Date: Fri, 22 Nov 2024 15:18:09 +0100
Subject: [PATCH 2/2] fmt

---
 chain/src/main.rs        | 2 +-
 chain/src/services/db.rs | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/chain/src/main.rs b/chain/src/main.rs
index ded8007..eac8a16 100644
--- a/chain/src/main.rs
+++ b/chain/src/main.rs
@@ -123,7 +123,7 @@ async fn run_migrations(app_state: &AppState) -> Result<(), MainError> {
             Ok(_) => {
                 return migration_res
                     .context("Failed to run db migrations")
-                    .into_db_error()
+                    .into_db_error();
             }
             Err(e) => {
                 tracing::debug!(
diff --git a/chain/src/services/db.rs b/chain/src/services/db.rs
index 6b98b80..1a596d8 100644
--- a/chain/src/services/db.rs
+++ b/chain/src/services/db.rs
@@ -36,7 +36,9 @@ pub async fn run_migrations(conn: Object) -> anyhow::Result<()> {
     conn.interact(|transaction_conn| {
         transaction_conn
             .run_pending_migrations(MIGRATIONS)
-            .map_err(|e| anyhow!("Failed to run db migrations: {}", e.to_string()))?;
+            .map_err(|e| {
+                anyhow!("Failed to run db migrations: {}", e.to_string())
+            })?;
         anyhow::Ok(())
     })
     .await