Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sql migration and optimize plugin for big servers #56

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/dev/unnm3d/rediseconomy/RedisEconomyPlugin.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ public void onEnable() {
this.getLogger().info("Hooked into Vault!");

if (settings().migrationEnabled) {
scheduler.runTaskLater(() ->
currenciesManager.migrateFromOfflinePlayers(getServer().getOfflinePlayers()), 100L);
scheduler.runTaskLaterAsynchronously(() ->
currenciesManager.migrate(), 100L);
} else {
currenciesManager.loadDefaultCurrency(this.vaultPlugin);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@

import java.io.*;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.*;
import java.util.concurrent.CompletableFuture;

@AllArgsConstructor
public class BackupRestoreCommand implements CommandExecutor, TabCompleter {
private static final long MAX_MEMORY_USAGE = 10L * 1024 * 1024 * 1024;
private static final int MEMORY_CHUNK_USAGE = 3;
private static final int LINE_SIZE_ESTIMATE = 130;

private final CurrenciesManager currenciesManager;
private final RedisEconomyPlugin plugin;

Expand All @@ -37,20 +39,51 @@ public boolean onCommand(@NotNull CommandSender sender, @NotNull Command command
Path userPath = Path.of(plugin.getDataFolder().getAbsolutePath(), args[0]);
switch (label) {
case "backup-economy" -> {
try (FileWriter fw = new FileWriter(userPath.normalize().toFile())) {
StringBuilder sb = new StringBuilder();
currenciesManager.getCurrencies().forEach(currency ->
currency.getAccounts().forEach((uuid, balance) ->
sb.append(currency.getCurrencyName())
.append(";")
.append(uuid.toString())
.append(";")
.append(currenciesManager.getUsernameFromUUIDCache(uuid))
.append(";")
.append(balance)
.append(System.getProperty("line.separator"))
));
fw.write(sb.toString());// currency;uuid;name;balance
final Map<UUID, String> names = new HashMap<>();
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you really need to check RAM requirements?
we are talking of megabytes not gigabytes.
i tested with like 40k entries.
for 400000 entries I calculated 57MB of RAM

for (Map.Entry<String, UUID> entry : currenciesManager.getNameUniqueIds().entrySet()) {
names.put(entry.getValue(), entry.getKey());
}

try (BufferedWriter writer = new BufferedWriter(new FileWriter(userPath.normalize().toFile(), true))) {
long memory = Math.min(Runtime.getRuntime().freeMemory(), MAX_MEMORY_USAGE);
int chunkSize = (int) (memory / MEMORY_CHUNK_USAGE) / LINE_SIZE_ESTIMATE;

for (Currency currency : currenciesManager.getCurrencies()) {
Map<UUID, Double> accounts = currency.getAccounts();
plugin.getLogger().info("[" + currency.getCurrencyName() + "] Total accounts: " + accounts.size());
if (accounts.isEmpty()) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok that's fair. but check empty balances too. by default Redis will set them to %starting_balance%

continue;
}

final Iterator<Map.Entry<UUID, Double>> iterator = accounts.entrySet().iterator();
int i;
for (i = 0; i < accounts.size(); i += chunkSize) {
final StringBuilder chunk = new StringBuilder();

int end = Math.min(i + chunkSize, accounts.size());
for (int j = i; j < end; j++) {
final Map.Entry<UUID, Double> entry = iterator.next();
final UUID uuid = entry.getKey();
final double balance = entry.getValue();
// currency;uuid;name;balance
chunk.append(currency.getCurrencyName())
.append(";")
.append(uuid.toString())
.append(";")
.append(names.getOrDefault(uuid, "null"))
.append(";")
.append(balance)
.append(System.lineSeparator());
}

writer.write(chunk.toString());
writer.flush();

plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + i + "/" + accounts.size());
}

plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + Math.min(i + chunkSize, accounts.size()) + "/" + accounts.size());
}
} catch (IOException e) {
e.printStackTrace();
}
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/dev/unnm3d/rediseconomy/config/Settings.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ public class Settings {
@Comment({"if true, migrates the bukkit offline uuids accounts to the default RedisEconomy currency",
"During the migration, the plugin will be disabled. Restart all RedisEconomy instances after the migration."})
public boolean migrationEnabled = false;
@Comment({"if enabled, the plugin will migrate economy data from sql database",
"During the migration, the plugin will be disabled. Restart all RedisEconomy instances after the migration."})
Rubenicos marked this conversation as resolved.
Show resolved Hide resolved
public SqlMigrateSettings migrationSql = new SqlMigrateSettings(false, "com.mysql.cj.jdbc.Driver", "jdbc:mysql://localhost:3306/database?useSSL=false", "root", "password", "economy", "name", "uuid", "money");
@Comment("Allow paying with percentage (ex: /pay player 10% sends 'player' 10% of the sender balance)")
public boolean allowPercentagePayments = true;
@Comment({"Leave password or user empty if you don't have a password or user",
Expand Down Expand Up @@ -52,4 +55,7 @@ public record CurrencySettings(String currencyName, String currencySingle, Strin
public record RedisSettings(String host, int port, String user, String password, int database, int timeout,
String clientName) {
}

public record SqlMigrateSettings(boolean enabled, String driver, String url, String username, String password, String table, String nameColumn, String uuidColumn, String moneyColumn) {
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@
import dev.unnm3d.rediseconomy.api.RedisEconomyAPI;
import dev.unnm3d.rediseconomy.config.ConfigManager;
import dev.unnm3d.rediseconomy.config.Settings;
import dev.unnm3d.rediseconomy.currency.migration.OfflinePlayerCurrencyMigration;
import dev.unnm3d.rediseconomy.currency.migration.SqlCurrencyMigration;
import dev.unnm3d.rediseconomy.redis.RedisKeys;
import dev.unnm3d.rediseconomy.redis.RedisManager;
import dev.unnm3d.rediseconomy.transaction.EconomyExchange;
import io.lettuce.core.ScoredValue;
import io.lettuce.core.pubsub.StatefulRedisPubSubConnection;
import lombok.Getter;
import net.milkbowl.vault.economy.Economy;
import org.bukkit.Bukkit;
import org.bukkit.OfflinePlayer;
import org.bukkit.entity.Player;
import org.bukkit.event.EventHandler;
import org.bukkit.event.Listener;
Expand Down Expand Up @@ -54,8 +54,8 @@ public CurrenciesManager(RedisManager redisManager, RedisEconomyPlugin plugin, C
this.configManager = configManager;
this.currencies = new HashMap<>();
try {
this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(1, TimeUnit.SECONDS);
this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(1, TimeUnit.SECONDS);
this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(10, TimeUnit.SECONDS);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why 10 seconds. if you have that latency it means your redis is down.
Redis isn't like MySQL, it doesn't take ages to load data.
I'm ok if you change it to 3 or 4 secs

this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(10, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new RuntimeException(e);
}
Expand Down Expand Up @@ -92,39 +92,15 @@ public void loadDefaultCurrency(Plugin vaultPlugin) {

/**
* Migrates the balances from the existent economy provider to the new one
* using the offline players
*
* @param offlinePlayers the offline players to migrate
*/
public void migrateFromOfflinePlayers(OfflinePlayer[] offlinePlayers) {
final Currency defaultCurrency = getDefaultCurrency();
RegisteredServiceProvider<Economy> existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class);
if (existentProvider == null) {
plugin.getLogger().severe("Vault economy provider not found!");
return;
}
plugin.getLogger().info("§aMigrating from " + existentProvider.getProvider().getName() + "...");

final List<ScoredValue<String>> balances = new ArrayList<>();
final Map<String, String> nameUniqueIds = new HashMap<>();
for (int i = 0; i < offlinePlayers.length; i++) {
final OfflinePlayer offlinePlayer = offlinePlayers[i];
try {
double bal = existentProvider.getProvider().getBalance(offlinePlayer);
balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString()));
if (offlinePlayer.getName() != null)
nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString());
defaultCurrency.updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal);
} catch (Exception e) {
e.printStackTrace();
}
if (i % 100 == 0) {
plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length);
}
public void migrate() {
final CurrencyMigration migration;
if (configManager.getSettings().migrationSql.enabled()) {
Rubenicos marked this conversation as resolved.
Show resolved Hide resolved
migration = new SqlCurrencyMigration(plugin, getDefaultCurrency());
} else {
migration = new OfflinePlayerCurrencyMigration(plugin, getDefaultCurrency());
}
defaultCurrency.updateBulkAccountsCloudCache(balances, nameUniqueIds);
plugin.getLogger().info("§aMigration completed!");
plugin.getLogger().info("§aRestart the server to apply the changes.");
migration.migrate();
configManager.getSettings().migrationEnabled = false;
configManager.saveConfigs();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ public EconomyResponse depositPlayer(@NotNull UUID playerUUID, @Nullable String
return new EconomyResponse(amount, getBalance(playerUUID), EconomyResponse.ResponseType.SUCCESS, null);
}

void updateAccountLocal(@NotNull UUID uuid, @Nullable String playerName, double balance) {
public void updateAccountLocal(@NotNull UUID uuid, @Nullable String playerName, double balance) {
Rubenicos marked this conversation as resolved.
Show resolved Hide resolved
if (playerName != null)
currenciesManager.updateNameUniqueId(playerName, uuid);
accounts.put(uuid, balance);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package dev.unnm3d.rediseconomy.currency;

import dev.unnm3d.rediseconomy.RedisEconomyPlugin;
import org.bukkit.Bukkit;

public abstract class CurrencyMigration {

protected final RedisEconomyPlugin plugin;
protected final Currency currency;

public CurrencyMigration(RedisEconomyPlugin plugin, Currency currency) {
this.plugin = plugin;
this.currency = currency;
}

public abstract String getProvider();

protected abstract boolean setup();

protected abstract void start();

public void migrate() {
if (!setup()) {
return;
}

Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aMigrating from " + getProvider() + "...");
Rubenicos marked this conversation as resolved.
Show resolved Hide resolved

start();

Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aMigration completed!");
Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aRestart the server to apply the changes.");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package dev.unnm3d.rediseconomy.currency.migration;

import dev.unnm3d.rediseconomy.RedisEconomyPlugin;
import dev.unnm3d.rediseconomy.currency.Currency;
import dev.unnm3d.rediseconomy.currency.CurrencyMigration;
import io.lettuce.core.ScoredValue;
import net.milkbowl.vault.economy.Economy;
import org.bukkit.Bukkit;
import org.bukkit.OfflinePlayer;
import org.bukkit.plugin.RegisteredServiceProvider;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class OfflinePlayerCurrencyMigration extends CurrencyMigration {

RegisteredServiceProvider<Economy> existentProvider;
Rubenicos marked this conversation as resolved.
Show resolved Hide resolved

public OfflinePlayerCurrencyMigration(RedisEconomyPlugin plugin, Currency currency) {
super(plugin, currency);
}

@Override
public String getProvider() {
return existentProvider.getProvider().getName();
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

existentProvider.getProvider() can be null

}

@Override
protected boolean setup() {
existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class);
if (existentProvider == null) {
plugin.getLogger().severe("Vault economy provider not found!");
return false;
}
return true;
}

@Override
protected void start() {
OfflinePlayer[] offlinePlayers = Bukkit.getOfflinePlayers();

final List<ScoredValue<String>> balances = new ArrayList<>();
final Map<String, String> nameUniqueIds = new HashMap<>();
for (int i = 0; i < offlinePlayers.length; i++) {
final OfflinePlayer offlinePlayer = offlinePlayers[i];
try {
double bal = existentProvider.getProvider().getBalance(offlinePlayer);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

existentProvider.getProvider() can be null

balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString()));
if (offlinePlayer.getName() != null)
nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString());
currency.updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal);
} catch (Exception e) {
e.printStackTrace();
}
if (i % 1000 == 0) {
plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length);
}
}
currency.updateBulkAccountsCloudCache(balances, nameUniqueIds);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package dev.unnm3d.rediseconomy.currency.migration;

import dev.unnm3d.rediseconomy.RedisEconomyPlugin;
import dev.unnm3d.rediseconomy.config.Settings;
import dev.unnm3d.rediseconomy.currency.Currency;
import dev.unnm3d.rediseconomy.currency.CurrencyMigration;
import io.lettuce.core.ScoredValue;

import java.sql.*;
import java.util.*;
import java.util.logging.Level;

public class SqlCurrencyMigration extends CurrencyMigration {

private Settings.SqlMigrateSettings sql;
private Connection connection;

public SqlCurrencyMigration(RedisEconomyPlugin plugin, Currency currency) {
super(plugin, currency);
}

@Override
public String getProvider() {
return "SQL DATABASE";
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you explain this please?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of returning a economy provider name like offline player migration, the SQL migration doesn't have that

}

@Override
protected boolean setup() {
sql = plugin.getConfigManager().getSettings().migrationSql;
try {
Class.forName(sql.driver());
} catch (ClassNotFoundException e) {
plugin.getLogger().warning("Cannot find SQL driver class: " + sql.driver());
}
try {
connection = DriverManager.getConnection(sql.url(), sql.username(), sql.password());
return true;
} catch (Throwable t) {
plugin.getLogger().log(Level.SEVERE, t, () -> "Cannot connect to SQL database");
return false;
}
}

@Override
protected void start() {
final List<ScoredValue<String>> balances = new ArrayList<>();
final Map<String, String> nameUniqueIds = new HashMap<>();

try (PreparedStatement count = connection.prepareStatement("SELECT COUNT(*) FROM `" + sql.table() + "`"); PreparedStatement stmt = connection.prepareStatement("SELECT ALL * FROM `" + sql.table() + "`")) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't it better to separate those 2 queries?

String total = "ALL";
try {
ResultSet resultSet = count.executeQuery();

if (resultSet.next()) {
int rowCount = resultSet.getInt(1);
total = String.valueOf(rowCount);
}
} catch (SQLException ignored) { }

plugin.getLogger().info("Total lines: " + total);

final ResultSet result = stmt.executeQuery();
int i = 0;
while (result.next()) {
i++;
try {
final String name = result.getString(sql.nameColumn());
final String uuid = result.getString(sql.uuidColumn());
if (uuid == null) continue;
final double money = result.getDouble(sql.moneyColumn());
if (money == 0) continue;

balances.add(ScoredValue.just(money, uuid));
if (name != null)
nameUniqueIds.put(name, uuid);
currency.updateAccountLocal(UUID.fromString(uuid), name == null ? uuid : name, money);
} catch (Exception e) {
e.printStackTrace();
Copy link
Owner

@Emibergo02 Emibergo02 May 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Send only a warning with the exception message or the user will be flooded by errors

}

if (i % 1000 == 0) {
plugin.getLogger().info("Progress: " + i + "/" + total);
}
}
} catch (SQLException e) {
e.printStackTrace();
}

try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
currency.updateBulkAccountsCloudCache(balances, nameUniqueIds);
}
}