Add sql migration and optimize plugin for big servers

src/main/java/dev/unnm3d/rediseconomy/RedisEconomyPlugin.java

@@ -92,8 +92,8 @@ public void onEnable() {
         this.getLogger().info("Hooked into Vault!");
 
         if (settings().migrationEnabled) {
-            scheduler.runTaskLater(() ->
-                    currenciesManager.migrateFromOfflinePlayers(getServer().getOfflinePlayers()), 100L);
+            scheduler.runTaskLaterAsynchronously(() ->
+                    currenciesManager.migrate(), 100L);
         } else {
             currenciesManager.loadDefaultCurrency(this.vaultPlugin);
         }

src/main/java/dev/unnm3d/rediseconomy/command/BackupRestoreCommand.java

@@ -13,13 +13,15 @@
 
 import java.io.*;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 
 @AllArgsConstructor
 public class BackupRestoreCommand implements CommandExecutor, TabCompleter {
+    private static final long MAX_MEMORY_USAGE = 10L * 1024 * 1024 * 1024;
+    private static final int MEMORY_CHUNK_USAGE = 3;
+    private static final int LINE_SIZE_ESTIMATE = 130;
+
     private final CurrenciesManager currenciesManager;
     private final RedisEconomyPlugin plugin;
 
@@ -37,20 +39,51 @@ public boolean onCommand(@NotNull CommandSender sender, @NotNull Command command
             Path userPath = Path.of(plugin.getDataFolder().getAbsolutePath(), args[0]);
             switch (label) {
                 case "backup-economy" -> {
-                    try (FileWriter fw = new FileWriter(userPath.normalize().toFile())) {
-                        StringBuilder sb = new StringBuilder();
-                        currenciesManager.getCurrencies().forEach(currency ->
-                                currency.getAccounts().forEach((uuid, balance) ->
-                                        sb.append(currency.getCurrencyName())
-                                                .append(";")
-                                                .append(uuid.toString())
-                                                .append(";")
-                                                .append(currenciesManager.getUsernameFromUUIDCache(uuid))
-                                                .append(";")
-                                                .append(balance)
-                                                .append(System.getProperty("line.separator"))
-                                ));
-                        fw.write(sb.toString());// currency;uuid;name;balance
+                    final Map<UUID, String> names = new HashMap<>();
+                    for (Map.Entry<String, UUID> entry : currenciesManager.getNameUniqueIds().entrySet()) {
+                        names.put(entry.getValue(), entry.getKey());
+                    }
+
+                    try (BufferedWriter writer = new BufferedWriter(new FileWriter(userPath.normalize().toFile(), true))) {
+                        long memory = Math.min(Runtime.getRuntime().freeMemory(), MAX_MEMORY_USAGE);
+                        int chunkSize = (int) (memory / MEMORY_CHUNK_USAGE) / LINE_SIZE_ESTIMATE;
+
+                        for (Currency currency : currenciesManager.getCurrencies()) {
+                            Map<UUID, Double> accounts = currency.getAccounts();
+                            plugin.getLogger().info("[" + currency.getCurrencyName() + "] Total accounts: " + accounts.size());
+                            if (accounts.isEmpty()) {
+                                continue;
+                            }
+
+                            final Iterator<Map.Entry<UUID, Double>> iterator = accounts.entrySet().iterator();
+                            int i;
+                            for (i = 0; i < accounts.size(); i += chunkSize) {
+                                final StringBuilder chunk = new StringBuilder();
+
+                                int end = Math.min(i + chunkSize, accounts.size());
+                                for (int j = i; j < end; j++) {
+                                    final Map.Entry<UUID, Double> entry = iterator.next();
+                                    final UUID uuid = entry.getKey();
+                                    final double balance = entry.getValue();
+                                    // currency;uuid;name;balance
+                                    chunk.append(currency.getCurrencyName())
+                                            .append(";")
+                                            .append(uuid.toString())
+                                            .append(";")
+                                            .append(names.getOrDefault(uuid, "null"))
+                                            .append(";")
+                                            .append(balance)
+                                            .append(System.lineSeparator());
+                                }
+
+                                writer.write(chunk.toString());
+                                writer.flush();
+
+                                plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + i + "/" + accounts.size());
+                            }
+
+                            plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + Math.min(i + chunkSize, accounts.size()) + "/" + accounts.size());
+                        }
                     } catch (IOException e) {
                         e.printStackTrace();
                     }

src/main/java/dev/unnm3d/rediseconomy/config/Langs.java

@@ -60,6 +60,8 @@ public final class Langs {
     public UnitSymbols unitSymbols = new UnitSymbols("k", "m", "b", "t", "q");
     public String backupRestoreFinished = "<green>Backup/restore file %file% finished!</green>";
     public String invalidPath = "<red>Invalid path!</red>";
+    public String migrationStart = "<green>Migrating from %provider%...</green>";
+    public String migrationCompleted = "<green>Migration completed! Restart the server to apply the changes.</green>";
 
 
     public record TransactionItem(

src/main/java/dev/unnm3d/rediseconomy/config/Settings.java

@@ -23,6 +23,9 @@ public class Settings {
     @Comment({"if true, migrates the bukkit offline uuids accounts to the default RedisEconomy currency",
             "During the migration, the plugin will be disabled. Restart all RedisEconomy instances after the migration."})
     public boolean migrationEnabled = false;
+    @Comment({"if enabled, the plugin will migrate economy data from sql database",
+            "During the migration, the plugin will be disabled. Restart all RedisEconomy instances after the migration."})
+    public SqlMigrateSettings sqlMigration = new SqlMigrateSettings(false, "com.mysql.cj.jdbc.Driver", "jdbc:mysql://localhost:3306/database?useSSL=false", "root", "password", "economy", "name", "uuid", "money");
     @Comment("Allow paying with percentage (ex: /pay player 10% sends 'player' 10% of the sender balance)")
     public boolean allowPercentagePayments = true;
     @Comment({"Leave password or user empty if you don't have a password or user",
@@ -52,4 +55,25 @@ public record CurrencySettings(String currencyName, String currencySingle, Strin
     public record RedisSettings(String host, int port, String user, String password, int database, int timeout,
                                 String clientName) {
     }
+
+    public record SqlMigrateSettings(
+            @Comment("Enable or not SQL migration")
+            boolean enabled,
+            @Comment("Sql driver to use, this is not mandatory due just make a safe-check before initializing the connection")
+            String driver,
+            @Comment("Connection URL, replace 'localhost:3306' and 'database' with your database information")
+            String url,
+            @Comment("Connection username")
+            String username,
+            @Comment("Connection password")
+            String password,
+            @Comment("This is the table on SQL database to get data from")
+            String table,
+            @Comment("Column name inside provided table to get player name")
+            String nameColumn,
+            @Comment("Column name inside provided table to get player unique id")
+            String uuidColumn,
+            @Comment("Column name inside provided table to get player balance")
+            String moneyColumn) {
+    };
 }

src/main/java/dev/unnm3d/rediseconomy/currency/CurrenciesManager.java

@@ -4,15 +4,15 @@
 import dev.unnm3d.rediseconomy.api.RedisEconomyAPI;
 import dev.unnm3d.rediseconomy.config.ConfigManager;
 import dev.unnm3d.rediseconomy.config.Settings;
+import dev.unnm3d.rediseconomy.currency.migration.OfflinePlayerCurrencyMigration;
+import dev.unnm3d.rediseconomy.currency.migration.SqlCurrencyMigration;
 import dev.unnm3d.rediseconomy.redis.RedisKeys;
 import dev.unnm3d.rediseconomy.redis.RedisManager;
 import dev.unnm3d.rediseconomy.transaction.EconomyExchange;
-import io.lettuce.core.ScoredValue;
 import io.lettuce.core.pubsub.StatefulRedisPubSubConnection;
 import lombok.Getter;
 import net.milkbowl.vault.economy.Economy;
 import org.bukkit.Bukkit;
-import org.bukkit.OfflinePlayer;
 import org.bukkit.entity.Player;
 import org.bukkit.event.EventHandler;
 import org.bukkit.event.Listener;
@@ -54,8 +54,8 @@ public CurrenciesManager(RedisManager redisManager, RedisEconomyPlugin plugin, C
         this.configManager = configManager;
         this.currencies = new HashMap<>();
         try {
-            this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(1, TimeUnit.SECONDS);
-            this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(1, TimeUnit.SECONDS);
+            this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(10, TimeUnit.SECONDS);
+            this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(10, TimeUnit.SECONDS);
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
             throw new RuntimeException(e);
         }
@@ -92,39 +92,15 @@ public void loadDefaultCurrency(Plugin vaultPlugin) {
 
     /**
      * Migrates the balances from the existent economy provider to the new one
-     * using the offline players
-     *
-     * @param offlinePlayers the offline players to migrate
      */
-    public void migrateFromOfflinePlayers(OfflinePlayer[] offlinePlayers) {
-        final Currency defaultCurrency = getDefaultCurrency();
-        RegisteredServiceProvider<Economy> existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class);
-        if (existentProvider == null) {
-            plugin.getLogger().severe("Vault economy provider not found!");
-            return;
-        }
-        plugin.getLogger().info("§aMigrating from " + existentProvider.getProvider().getName() + "...");
-
-        final List<ScoredValue<String>> balances = new ArrayList<>();
-        final Map<String, String> nameUniqueIds = new HashMap<>();
-        for (int i = 0; i < offlinePlayers.length; i++) {
-            final OfflinePlayer offlinePlayer = offlinePlayers[i];
-            try {
-                double bal = existentProvider.getProvider().getBalance(offlinePlayer);
-                balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString()));
-                if (offlinePlayer.getName() != null)
-                    nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString());
-                defaultCurrency.updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-            if (i % 100 == 0) {
-                plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length);
-            }
+    public void migrate() {
+        final CurrencyMigration migration;
+        if (configManager.getSettings().sqlMigration.enabled()) {
+            migration = new SqlCurrencyMigration(plugin, getDefaultCurrency());
+        } else {
+            migration = new OfflinePlayerCurrencyMigration(plugin, getDefaultCurrency());
         }
-        defaultCurrency.updateBulkAccountsCloudCache(balances, nameUniqueIds);
-        plugin.getLogger().info("§aMigration completed!");
-        plugin.getLogger().info("§aRestart the server to apply the changes.");
+        migration.migrate();
         configManager.getSettings().migrationEnabled = false;
         configManager.saveConfigs();
     }

src/main/java/dev/unnm3d/rediseconomy/currency/CurrencyMigration.java

@@ -0,0 +1,41 @@
+package dev.unnm3d.rediseconomy.currency;
+
+import dev.unnm3d.rediseconomy.RedisEconomyPlugin;
+import org.bukkit.Bukkit;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.UUID;
+
+public abstract class CurrencyMigration {
+
+    protected final RedisEconomyPlugin plugin;
+    protected final Currency currency;
+
+    public CurrencyMigration(RedisEconomyPlugin plugin, Currency currency) {
+        this.plugin = plugin;
+        this.currency = currency;
+    }
+
+    public abstract String getProvider();
+
+    protected abstract boolean setup();
+
+    protected abstract void start();
+
+    public void migrate() {
+        if (!setup()) {
+            return;
+        }
+
+        plugin.langs().send(Bukkit.getConsoleSender(), plugin.langs().migrationStart.replace("%provider%", getProvider()));
+
+        start();
+
+        plugin.langs().send(Bukkit.getConsoleSender(), plugin.langs().migrationCompleted);
+    }
+
+    protected void updateAccountLocal(@NotNull UUID uuid, @Nullable String playerName, double balance) {
+        currency.updateAccountLocal(uuid, playerName, balance);
+    }
+}

src/main/java/dev/unnm3d/rediseconomy/currency/migration/OfflinePlayerCurrencyMigration.java

@@ -0,0 +1,63 @@
+package dev.unnm3d.rediseconomy.currency.migration;
+
+import dev.unnm3d.rediseconomy.RedisEconomyPlugin;
+import dev.unnm3d.rediseconomy.currency.Currency;
+import dev.unnm3d.rediseconomy.currency.CurrencyMigration;
+import io.lettuce.core.ScoredValue;
+import net.milkbowl.vault.economy.Economy;
+import org.bukkit.Bukkit;
+import org.bukkit.OfflinePlayer;
+import org.bukkit.plugin.RegisteredServiceProvider;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class OfflinePlayerCurrencyMigration extends CurrencyMigration {
+
+    private RegisteredServiceProvider<Economy> existentProvider;
+
+    public OfflinePlayerCurrencyMigration(RedisEconomyPlugin plugin, Currency currency) {
+        super(plugin, currency);
+    }
+
+    @Override
+    public String getProvider() {
+        return existentProvider.getProvider().getName();
+    }
+
+    @Override
+    protected boolean setup() {
+        existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class);
+        if (existentProvider == null) {
+            plugin.getLogger().severe("Vault economy provider not found!");
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    protected void start() {
+        OfflinePlayer[] offlinePlayers = Bukkit.getOfflinePlayers();
+
+        final List<ScoredValue<String>> balances = new ArrayList<>();
+        final Map<String, String> nameUniqueIds = new HashMap<>();
+        for (int i = 0; i < offlinePlayers.length; i++) {
+            final OfflinePlayer offlinePlayer = offlinePlayers[i];
+            try {
+                double bal = existentProvider.getProvider().getBalance(offlinePlayer);
+                balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString()));
+                if (offlinePlayer.getName() != null)
+                    nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString());
+                updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            if (i % 1000 == 0) {
+                plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length);
+            }
+        }
+        currency.updateBulkAccountsCloudCache(balances, nameUniqueIds);
+    }
+}