-
Notifications
You must be signed in to change notification settings - Fork 6
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add sql migration and optimize plugin for big servers #56
base: main
Are you sure you want to change the base?
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,13 +13,15 @@ | |
|
||
import java.io.*; | ||
import java.nio.file.Path; | ||
import java.util.ArrayList; | ||
import java.util.HashMap; | ||
import java.util.List; | ||
import java.util.*; | ||
import java.util.concurrent.CompletableFuture; | ||
|
||
@AllArgsConstructor | ||
public class BackupRestoreCommand implements CommandExecutor, TabCompleter { | ||
private static final long MAX_MEMORY_USAGE = 10L * 1024 * 1024 * 1024; | ||
private static final int MEMORY_CHUNK_USAGE = 3; | ||
private static final int LINE_SIZE_ESTIMATE = 130; | ||
|
||
private final CurrenciesManager currenciesManager; | ||
private final RedisEconomyPlugin plugin; | ||
|
||
|
@@ -37,20 +39,51 @@ public boolean onCommand(@NotNull CommandSender sender, @NotNull Command command | |
Path userPath = Path.of(plugin.getDataFolder().getAbsolutePath(), args[0]); | ||
switch (label) { | ||
case "backup-economy" -> { | ||
try (FileWriter fw = new FileWriter(userPath.normalize().toFile())) { | ||
StringBuilder sb = new StringBuilder(); | ||
currenciesManager.getCurrencies().forEach(currency -> | ||
currency.getAccounts().forEach((uuid, balance) -> | ||
sb.append(currency.getCurrencyName()) | ||
.append(";") | ||
.append(uuid.toString()) | ||
.append(";") | ||
.append(currenciesManager.getUsernameFromUUIDCache(uuid)) | ||
.append(";") | ||
.append(balance) | ||
.append(System.getProperty("line.separator")) | ||
)); | ||
fw.write(sb.toString());// currency;uuid;name;balance | ||
final Map<UUID, String> names = new HashMap<>(); | ||
for (Map.Entry<String, UUID> entry : currenciesManager.getNameUniqueIds().entrySet()) { | ||
names.put(entry.getValue(), entry.getKey()); | ||
} | ||
|
||
try (BufferedWriter writer = new BufferedWriter(new FileWriter(userPath.normalize().toFile(), true))) { | ||
long memory = Math.min(Runtime.getRuntime().freeMemory(), MAX_MEMORY_USAGE); | ||
int chunkSize = (int) (memory / MEMORY_CHUNK_USAGE) / LINE_SIZE_ESTIMATE; | ||
|
||
for (Currency currency : currenciesManager.getCurrencies()) { | ||
Map<UUID, Double> accounts = currency.getAccounts(); | ||
plugin.getLogger().info("[" + currency.getCurrencyName() + "] Total accounts: " + accounts.size()); | ||
if (accounts.isEmpty()) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok that's fair. but check empty balances too. by default Redis will set them to %starting_balance% |
||
continue; | ||
} | ||
|
||
final Iterator<Map.Entry<UUID, Double>> iterator = accounts.entrySet().iterator(); | ||
int i; | ||
for (i = 0; i < accounts.size(); i += chunkSize) { | ||
final StringBuilder chunk = new StringBuilder(); | ||
|
||
int end = Math.min(i + chunkSize, accounts.size()); | ||
for (int j = i; j < end; j++) { | ||
final Map.Entry<UUID, Double> entry = iterator.next(); | ||
final UUID uuid = entry.getKey(); | ||
final double balance = entry.getValue(); | ||
// currency;uuid;name;balance | ||
chunk.append(currency.getCurrencyName()) | ||
.append(";") | ||
.append(uuid.toString()) | ||
.append(";") | ||
.append(names.getOrDefault(uuid, "null")) | ||
.append(";") | ||
.append(balance) | ||
.append(System.lineSeparator()); | ||
} | ||
|
||
writer.write(chunk.toString()); | ||
writer.flush(); | ||
|
||
plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + i + "/" + accounts.size()); | ||
} | ||
|
||
plugin.getLogger().info("[" + currency.getCurrencyName() + "] Progress: " + Math.min(i + chunkSize, accounts.size()) + "/" + accounts.size()); | ||
} | ||
} catch (IOException e) { | ||
e.printStackTrace(); | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,15 +4,15 @@ | |
import dev.unnm3d.rediseconomy.api.RedisEconomyAPI; | ||
import dev.unnm3d.rediseconomy.config.ConfigManager; | ||
import dev.unnm3d.rediseconomy.config.Settings; | ||
import dev.unnm3d.rediseconomy.currency.migration.OfflinePlayerCurrencyMigration; | ||
import dev.unnm3d.rediseconomy.currency.migration.SqlCurrencyMigration; | ||
import dev.unnm3d.rediseconomy.redis.RedisKeys; | ||
import dev.unnm3d.rediseconomy.redis.RedisManager; | ||
import dev.unnm3d.rediseconomy.transaction.EconomyExchange; | ||
import io.lettuce.core.ScoredValue; | ||
import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; | ||
import lombok.Getter; | ||
import net.milkbowl.vault.economy.Economy; | ||
import org.bukkit.Bukkit; | ||
import org.bukkit.OfflinePlayer; | ||
import org.bukkit.entity.Player; | ||
import org.bukkit.event.EventHandler; | ||
import org.bukkit.event.Listener; | ||
|
@@ -54,8 +54,8 @@ public CurrenciesManager(RedisManager redisManager, RedisEconomyPlugin plugin, C | |
this.configManager = configManager; | ||
this.currencies = new HashMap<>(); | ||
try { | ||
this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(1, TimeUnit.SECONDS); | ||
this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(1, TimeUnit.SECONDS); | ||
this.nameUniqueIds = loadRedisNameUniqueIds().toCompletableFuture().get(10, TimeUnit.SECONDS); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why 10 seconds. if you have that latency it means your redis is down. |
||
this.lockedAccounts = loadLockedAccounts().toCompletableFuture().get(10, TimeUnit.SECONDS); | ||
} catch (InterruptedException | ExecutionException | TimeoutException e) { | ||
throw new RuntimeException(e); | ||
} | ||
|
@@ -92,39 +92,15 @@ public void loadDefaultCurrency(Plugin vaultPlugin) { | |
|
||
/** | ||
* Migrates the balances from the existent economy provider to the new one | ||
* using the offline players | ||
* | ||
* @param offlinePlayers the offline players to migrate | ||
*/ | ||
public void migrateFromOfflinePlayers(OfflinePlayer[] offlinePlayers) { | ||
final Currency defaultCurrency = getDefaultCurrency(); | ||
RegisteredServiceProvider<Economy> existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class); | ||
if (existentProvider == null) { | ||
plugin.getLogger().severe("Vault economy provider not found!"); | ||
return; | ||
} | ||
plugin.getLogger().info("§aMigrating from " + existentProvider.getProvider().getName() + "..."); | ||
|
||
final List<ScoredValue<String>> balances = new ArrayList<>(); | ||
final Map<String, String> nameUniqueIds = new HashMap<>(); | ||
for (int i = 0; i < offlinePlayers.length; i++) { | ||
final OfflinePlayer offlinePlayer = offlinePlayers[i]; | ||
try { | ||
double bal = existentProvider.getProvider().getBalance(offlinePlayer); | ||
balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString())); | ||
if (offlinePlayer.getName() != null) | ||
nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString()); | ||
defaultCurrency.updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal); | ||
} catch (Exception e) { | ||
e.printStackTrace(); | ||
} | ||
if (i % 100 == 0) { | ||
plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length); | ||
} | ||
public void migrate() { | ||
final CurrencyMigration migration; | ||
if (configManager.getSettings().migrationSql.enabled()) { | ||
Rubenicos marked this conversation as resolved.
Show resolved
Hide resolved
|
||
migration = new SqlCurrencyMigration(plugin, getDefaultCurrency()); | ||
} else { | ||
migration = new OfflinePlayerCurrencyMigration(plugin, getDefaultCurrency()); | ||
} | ||
defaultCurrency.updateBulkAccountsCloudCache(balances, nameUniqueIds); | ||
plugin.getLogger().info("§aMigration completed!"); | ||
plugin.getLogger().info("§aRestart the server to apply the changes."); | ||
migration.migrate(); | ||
configManager.getSettings().migrationEnabled = false; | ||
configManager.saveConfigs(); | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package dev.unnm3d.rediseconomy.currency; | ||
|
||
import dev.unnm3d.rediseconomy.RedisEconomyPlugin; | ||
import org.bukkit.Bukkit; | ||
|
||
public abstract class CurrencyMigration { | ||
|
||
protected final RedisEconomyPlugin plugin; | ||
protected final Currency currency; | ||
|
||
public CurrencyMigration(RedisEconomyPlugin plugin, Currency currency) { | ||
this.plugin = plugin; | ||
this.currency = currency; | ||
} | ||
|
||
public abstract String getProvider(); | ||
|
||
protected abstract boolean setup(); | ||
|
||
protected abstract void start(); | ||
|
||
public void migrate() { | ||
if (!setup()) { | ||
return; | ||
} | ||
|
||
Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aMigrating from " + getProvider() + "..."); | ||
Rubenicos marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
start(); | ||
|
||
Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aMigration completed!"); | ||
Bukkit.getConsoleSender().sendMessage("[RedisEconomy] §aRestart the server to apply the changes."); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
package dev.unnm3d.rediseconomy.currency.migration; | ||
|
||
import dev.unnm3d.rediseconomy.RedisEconomyPlugin; | ||
import dev.unnm3d.rediseconomy.currency.Currency; | ||
import dev.unnm3d.rediseconomy.currency.CurrencyMigration; | ||
import io.lettuce.core.ScoredValue; | ||
import net.milkbowl.vault.economy.Economy; | ||
import org.bukkit.Bukkit; | ||
import org.bukkit.OfflinePlayer; | ||
import org.bukkit.plugin.RegisteredServiceProvider; | ||
|
||
import java.util.ArrayList; | ||
import java.util.HashMap; | ||
import java.util.List; | ||
import java.util.Map; | ||
|
||
public class OfflinePlayerCurrencyMigration extends CurrencyMigration { | ||
|
||
RegisteredServiceProvider<Economy> existentProvider; | ||
Rubenicos marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
public OfflinePlayerCurrencyMigration(RedisEconomyPlugin plugin, Currency currency) { | ||
super(plugin, currency); | ||
} | ||
|
||
@Override | ||
public String getProvider() { | ||
return existentProvider.getProvider().getName(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. existentProvider.getProvider() can be null |
||
} | ||
|
||
@Override | ||
protected boolean setup() { | ||
existentProvider = plugin.getServer().getServicesManager().getRegistration(Economy.class); | ||
if (existentProvider == null) { | ||
plugin.getLogger().severe("Vault economy provider not found!"); | ||
return false; | ||
} | ||
return true; | ||
} | ||
|
||
@Override | ||
protected void start() { | ||
OfflinePlayer[] offlinePlayers = Bukkit.getOfflinePlayers(); | ||
|
||
final List<ScoredValue<String>> balances = new ArrayList<>(); | ||
final Map<String, String> nameUniqueIds = new HashMap<>(); | ||
for (int i = 0; i < offlinePlayers.length; i++) { | ||
final OfflinePlayer offlinePlayer = offlinePlayers[i]; | ||
try { | ||
double bal = existentProvider.getProvider().getBalance(offlinePlayer); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. existentProvider.getProvider() can be null |
||
balances.add(ScoredValue.just(bal, offlinePlayer.getUniqueId().toString())); | ||
if (offlinePlayer.getName() != null) | ||
nameUniqueIds.put(offlinePlayer.getName(), offlinePlayer.getUniqueId().toString()); | ||
currency.updateAccountLocal(offlinePlayer.getUniqueId(), offlinePlayer.getName() == null ? offlinePlayer.getUniqueId().toString() : offlinePlayer.getName(), bal); | ||
} catch (Exception e) { | ||
e.printStackTrace(); | ||
} | ||
if (i % 1000 == 0) { | ||
plugin.getLogger().info("Progress: " + i + "/" + offlinePlayers.length); | ||
} | ||
} | ||
currency.updateBulkAccountsCloudCache(balances, nameUniqueIds); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
package dev.unnm3d.rediseconomy.currency.migration; | ||
|
||
import dev.unnm3d.rediseconomy.RedisEconomyPlugin; | ||
import dev.unnm3d.rediseconomy.config.Settings; | ||
import dev.unnm3d.rediseconomy.currency.Currency; | ||
import dev.unnm3d.rediseconomy.currency.CurrencyMigration; | ||
import io.lettuce.core.ScoredValue; | ||
|
||
import java.sql.*; | ||
import java.util.*; | ||
import java.util.logging.Level; | ||
|
||
public class SqlCurrencyMigration extends CurrencyMigration { | ||
|
||
private Settings.SqlMigrateSettings sql; | ||
private Connection connection; | ||
|
||
public SqlCurrencyMigration(RedisEconomyPlugin plugin, Currency currency) { | ||
super(plugin, currency); | ||
} | ||
|
||
@Override | ||
public String getProvider() { | ||
return "SQL DATABASE"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you explain this please? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Instead of returning a economy provider name like offline player migration, the SQL migration doesn't have that |
||
} | ||
|
||
@Override | ||
protected boolean setup() { | ||
sql = plugin.getConfigManager().getSettings().migrationSql; | ||
try { | ||
Class.forName(sql.driver()); | ||
} catch (ClassNotFoundException e) { | ||
plugin.getLogger().warning("Cannot find SQL driver class: " + sql.driver()); | ||
} | ||
try { | ||
connection = DriverManager.getConnection(sql.url(), sql.username(), sql.password()); | ||
return true; | ||
} catch (Throwable t) { | ||
plugin.getLogger().log(Level.SEVERE, t, () -> "Cannot connect to SQL database"); | ||
return false; | ||
} | ||
} | ||
|
||
@Override | ||
protected void start() { | ||
final List<ScoredValue<String>> balances = new ArrayList<>(); | ||
final Map<String, String> nameUniqueIds = new HashMap<>(); | ||
|
||
try (PreparedStatement count = connection.prepareStatement("SELECT COUNT(*) FROM `" + sql.table() + "`"); PreparedStatement stmt = connection.prepareStatement("SELECT ALL * FROM `" + sql.table() + "`")) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Isn't it better to separate those 2 queries? |
||
String total = "ALL"; | ||
try { | ||
ResultSet resultSet = count.executeQuery(); | ||
|
||
if (resultSet.next()) { | ||
int rowCount = resultSet.getInt(1); | ||
total = String.valueOf(rowCount); | ||
} | ||
} catch (SQLException ignored) { } | ||
|
||
plugin.getLogger().info("Total lines: " + total); | ||
|
||
final ResultSet result = stmt.executeQuery(); | ||
int i = 0; | ||
while (result.next()) { | ||
i++; | ||
try { | ||
final String name = result.getString(sql.nameColumn()); | ||
final String uuid = result.getString(sql.uuidColumn()); | ||
if (uuid == null) continue; | ||
final double money = result.getDouble(sql.moneyColumn()); | ||
if (money == 0) continue; | ||
|
||
balances.add(ScoredValue.just(money, uuid)); | ||
if (name != null) | ||
nameUniqueIds.put(name, uuid); | ||
currency.updateAccountLocal(UUID.fromString(uuid), name == null ? uuid : name, money); | ||
} catch (Exception e) { | ||
e.printStackTrace(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Send only a warning with the exception message or the user will be flooded by errors |
||
} | ||
|
||
if (i % 1000 == 0) { | ||
plugin.getLogger().info("Progress: " + i + "/" + total); | ||
} | ||
} | ||
} catch (SQLException e) { | ||
e.printStackTrace(); | ||
} | ||
|
||
try { | ||
connection.close(); | ||
} catch (SQLException e) { | ||
e.printStackTrace(); | ||
} | ||
currency.updateBulkAccountsCloudCache(balances, nameUniqueIds); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you really need to check RAM requirements?
we are talking of megabytes not gigabytes.
i tested with like 40k entries.
for 400000 entries I calculated 57MB of RAM