From 4fdeeabb52171abf4598919f04063fcda9e0504d Mon Sep 17 00:00:00 2001 From: Daniel Walsh Date: Sun, 11 Feb 2024 23:07:38 +0000 Subject: [PATCH] [WIP] Binary storage for player data --- pom.xml | 8 +- .../slimefun4/implementation/Slimefun.java | 18 ++- .../implementation/StartupWarnings.java | 14 ++ .../storage/backend/binary/BinaryStorage.java | 151 ++++++++++++++++++ .../storage/backend/binary/CommonKeys.java | 17 ++ .../binary/serializers/BinarySerialize.java | 10 ++ .../serializers/LocationSerializer.java | 60 +++++++ .../binary/serializers/UUIDSerializer.java | 43 +++++ src/main/resources/config.yml | 3 + 9 files changed, 319 insertions(+), 5 deletions(-) create mode 100644 src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/BinaryStorage.java create mode 100644 src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/CommonKeys.java create mode 100644 src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/BinarySerialize.java create mode 100644 src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/LocationSerializer.java create mode 100644 src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/UUIDSerializer.java diff --git a/pom.xml b/pom.xml index 411d3b015c..93380fb2e0 100644 --- a/pom.xml +++ b/pom.xml @@ -347,9 +347,9 @@ - com.github.baked-libs.dough + io.github.baked-libs dough-api - 1108163a49 + 1.3.0_local_nbt_v7 compile @@ -528,10 +528,12 @@ 2.6 compile + + org.spigotmc spigot-api - ${spigot.version}-R0.1-SNAPSHOT + ${spigot.version}-R0.1-20240209.081002-77 provided diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/Slimefun.java b/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/Slimefun.java index 28233ea741..3b678a3977 100644 --- a/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/Slimefun.java +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/Slimefun.java @@ -15,6 +15,7 @@ import javax.annotation.Nullable; import io.github.thebusybiscuit.slimefun4.storage.Storage; +import io.github.thebusybiscuit.slimefun4.storage.backend.binary.BinaryStorage; import io.github.thebusybiscuit.slimefun4.storage.backend.legacy.LegacyStorage; import org.apache.commons.lang.Validate; @@ -306,8 +307,21 @@ private void onPluginStart() { networkManager = new NetworkManager(networkSize, config.getBoolean("networks.enable-visualizer"), config.getBoolean("networks.delete-excess-items")); // Data storage - playerStorage = new LegacyStorage(); - logger.log(Level.INFO, "Using legacy storage for player data"); + String storageBackend = config.getString("storage.player-data"); + + if (storageBackend.equals("legacy")) { + playerStorage = new LegacyStorage(); + logger.info("Using legacy storage for player data"); + } else if (storageBackend.equals("binary")) { + playerStorage = new BinaryStorage(); + StartupWarnings.experimentalStorage(logger, storageBackend); + + // TODO(future): Run migration if needed + } else { + playerStorage = new LegacyStorage(); + logger.warning("Unknown storage backend for player data: " + storageBackend); + logger.warning("Defaulting to legacy storage instead"); + } // Setting up bStats new Thread(metricsService::start, "Slimefun Metrics").start(); diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/StartupWarnings.java b/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/StartupWarnings.java index 77b3a7479a..2db3339721 100644 --- a/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/StartupWarnings.java +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/implementation/StartupWarnings.java @@ -73,4 +73,18 @@ static void oldJavaVersion(Logger logger, int recommendedJavaVersion) { logger.log(Level.WARNING, BORDER); } + static void experimentalStorage(Logger logger, String backend) { + logger.log(Level.WARNING, BORDER); + logger.log(Level.WARNING, PREFIX + "You have enabled an experimental storage backend!"); + logger.log(Level.WARNING, PREFIX); + logger.log(Level.WARNING, PREFIX + "\"{0}\" storage is still in development.", backend); + logger.log(Level.WARNING, PREFIX + "It might not work as expected and it might"); + logger.log(Level.WARNING, PREFIX + "break your Slimefun data. Use at your own risk!"); + logger.log(Level.WARNING, PREFIX); + logger.log(Level.WARNING, PREFIX + "You can revert at any time but be aware you will"); + logger.log(Level.WARNING, PREFIX + "lose progress that happened during the time you switched."); + logger.log(Level.WARNING, PREFIX); + logger.log(Level.WARNING, PREFIX + "If you encounter any issues, please report them to us!"); + logger.log(Level.WARNING, BORDER); + } } diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/BinaryStorage.java b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/BinaryStorage.java new file mode 100644 index 0000000000..dc555b49f0 --- /dev/null +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/BinaryStorage.java @@ -0,0 +1,151 @@ +package io.github.thebusybiscuit.slimefun4.storage.backend.binary; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import io.github.thebusybiscuit.slimefun4.core.debug.Debug; +import io.github.thebusybiscuit.slimefun4.core.debug.TestCase; +import org.bukkit.Location; +import org.bukkit.NamespacedKey; + +import io.github.bakedlibs.dough.nbt.TagType; +import io.github.bakedlibs.dough.nbt.streams.CompressionType; +import io.github.bakedlibs.dough.nbt.streams.TagInputStream; +import io.github.bakedlibs.dough.nbt.streams.TagOutputStream; +import io.github.bakedlibs.dough.nbt.tags.CompoundTag; +import io.github.bakedlibs.dough.nbt.tags.ListTag; +import io.github.bakedlibs.dough.nbt.tags.StringTag; +import io.github.bakedlibs.dough.nbt.tags.Tag; +import io.github.thebusybiscuit.slimefun4.api.gps.Waypoint; +import io.github.thebusybiscuit.slimefun4.api.player.PlayerBackpack; +import io.github.thebusybiscuit.slimefun4.api.researches.Research; +import io.github.thebusybiscuit.slimefun4.implementation.Slimefun; +import io.github.thebusybiscuit.slimefun4.storage.Storage; +import io.github.thebusybiscuit.slimefun4.storage.backend.binary.serializers.LocationSerializer; +import io.github.thebusybiscuit.slimefun4.storage.data.PlayerData; + +public class BinaryStorage implements Storage { + + private static final NamespacedKey PLAYER_DATA = new NamespacedKey(Slimefun.instance(), "player_data"); + private static final NamespacedKey RESEARCHES_KEY = new NamespacedKey(Slimefun.instance(), "researches"); + private static final NamespacedKey BACKPACKS_KEY = new NamespacedKey(Slimefun.instance(), "backpacks"); + private static final NamespacedKey WAYPOINTS_KEY = new NamespacedKey(Slimefun.instance(), "waypoints"); + + // TODO: Move to lz4 or zstd + private static final CompressionType compression = CompressionType.GZIP; + + /* + * Structure as JSON: + * { + * "slimefun:researches": ["slimefun:some_research", "slimefun:some_other_research"], + * "slimefun:backpacks": [ + * { + * "id": x, + * "item": y + * } + * ], + * "slimefun:waypoints": { + * "": { + * "name": "", + * "location": {} + * } + * } + * } + */ + + @Override + public PlayerData loadPlayerData(UUID uuid) { + Debug.log(TestCase.PLAYER_PROFILE_DATA, "Loading player data from binary storage for {}", uuid); + File file = new File("data-storage/Slimefun/Players/" + uuid + ".dat"); + + if (!file.exists()) { + return new PlayerData(Set.of(), Map.of(), Set.of()); + } + + CompoundTag root; + try { + try (TagInputStream stream = new TagInputStream(new FileInputStream(file), compression)) { + root = (CompoundTag) stream.readTag(); + } + } catch(IOException e) { + throw new IllegalStateException("Failed to read Player data for " + uuid, e); + } + + // Load researches + ListTag list = root.getList(RESEARCHES_KEY, TagType.STRING); + + Set researches = new HashSet<>(); + for (Research research : Slimefun.getRegistry().getResearches()) { + for (StringTag tag : list) { + if (tag.getValue().equals(research.getKey().toString())) { + researches.add(research); + } + } + } + + // Load backpacks + Map backpacks = Map.of(); + CompoundTag backpackMap = root.getCompound(BACKPACKS_KEY); + // TODO: Item serialization + + // Load waypoints + Set waypoints = new HashSet<>(); + CompoundTag waypointMap = root.getCompound(WAYPOINTS_KEY); + for (Map.Entry> key : waypointMap) { + CompoundTag waypoint = (CompoundTag) key.getValue(); + + String name = waypoint.getString(CommonKeys.NAME); + Location location = LocationSerializer.INSTANCE.deserialize(waypoint.getCompound(CommonKeys.LOCATION)); + + waypoints.add(new Waypoint(uuid, key.getKey().getKey(), location, name)); + } + + return new PlayerData(researches, backpacks, waypoints); + } + + @Override + public void savePlayerData(UUID uuid, PlayerData data) { + Debug.log(TestCase.PLAYER_PROFILE_DATA, "Saving player data from binary storage for {}", uuid); + File file = new File("data-storage/Slimefun/Players/" + uuid + ".dat"); + + CompoundTag root = new CompoundTag(PLAYER_DATA); + + // Save researches + ListTag list = new ListTag<>(); + for (Research research : data.getResearches()) { + list.add(new StringTag(research.getKey().toString())); + } + + root.putList(RESEARCHES_KEY, list); + + // Save backpacks + CompoundTag backpackMap = new CompoundTag(); + + // Save waypoints + CompoundTag waypointMap = new CompoundTag(); + for (Waypoint waypoint : data.getWaypoints()) { + CompoundTag waypointTag = new CompoundTag(); + + waypointTag.putString(CommonKeys.ID, waypoint.getId()); + waypointTag.putString(CommonKeys.NAME, waypoint.getName()); + waypointTag.putCompound(CommonKeys.LOCATION, LocationSerializer.INSTANCE.serialize(waypoint.getLocation())); + + waypointMap.putCompound(new NamespacedKey(Slimefun.instance(), waypoint.getId()), waypointTag); + } + root.putCompound(WAYPOINTS_KEY, waypointMap); + + try { + try (TagOutputStream stream = new TagOutputStream(new FileOutputStream(file), compression)) { + stream.writeTag(root); + } + } catch(IOException e) { + throw new IllegalStateException("Failed to read Player data for " + uuid, e); + } + } +} diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/CommonKeys.java b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/CommonKeys.java new file mode 100644 index 0000000000..2fb3ef5e15 --- /dev/null +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/CommonKeys.java @@ -0,0 +1,17 @@ +package io.github.thebusybiscuit.slimefun4.storage.backend.binary; + +import org.bukkit.NamespacedKey; + +import io.github.thebusybiscuit.slimefun4.implementation.Slimefun; + +public class CommonKeys { + + public static final NamespacedKey ID = new NamespacedKey(Slimefun.instance(), "id"); + public static final NamespacedKey NAME = new NamespacedKey(Slimefun.instance(), "name"); + // Location stuff + public static final NamespacedKey LOCATION = new NamespacedKey(Slimefun.instance(), "location"); + public static final NamespacedKey WORLD = new NamespacedKey(Slimefun.instance(), "world"); + public static final NamespacedKey POSITION = new NamespacedKey(Slimefun.instance(), "position"); + public static final NamespacedKey PITCH = new NamespacedKey(Slimefun.instance(), "pitch"); + public static final NamespacedKey YAW = new NamespacedKey(Slimefun.instance(), "yaw"); +} diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/BinarySerialize.java b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/BinarySerialize.java new file mode 100644 index 0000000000..67d6a16601 --- /dev/null +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/BinarySerialize.java @@ -0,0 +1,10 @@ +package io.github.thebusybiscuit.slimefun4.storage.backend.binary.serializers; + +import io.github.bakedlibs.dough.nbt.tags.Tag; + +public interface BinarySerialize> { + + public T serialize(O obj); + + public O deserialize(T obj); +} diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/LocationSerializer.java b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/LocationSerializer.java new file mode 100644 index 0000000000..cac6da8f17 --- /dev/null +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/LocationSerializer.java @@ -0,0 +1,60 @@ +package io.github.thebusybiscuit.slimefun4.storage.backend.binary.serializers; + +import java.util.UUID; + +import org.bukkit.Bukkit; +import org.bukkit.Location; + +import io.github.bakedlibs.dough.blocks.BlockPosition; +import io.github.bakedlibs.dough.nbt.tags.CompoundTag; +import io.github.bakedlibs.dough.nbt.tags.IntArrayTag; +import io.github.thebusybiscuit.slimefun4.storage.backend.binary.CommonKeys; + +public class LocationSerializer implements BinarySerialize { + + public static final LocationSerializer INSTANCE = new LocationSerializer(); + + private LocationSerializer() {} + + @Override + public CompoundTag serialize(Location location) { + CompoundTag tag = new CompoundTag(); + + if (location.getWorld() != null) { + tag.put(CommonKeys.WORLD, UUIDSerializer.INSTANCE.serialize(location.getWorld().getUID())); + } + + tag.putDouble(CommonKeys.POSITION, BlockPosition.getAsLong(location)); + + if (location.getPitch() != 0.0F) { + tag.putFloat(CommonKeys.PITCH, location.getPitch()); + } + if (location.getYaw() != 0.0F) { + tag.putFloat(CommonKeys.YAW, location.getYaw()); + } + + return tag; + } + + @Override + public Location deserialize(CompoundTag location) { + UUID worldUuid = null; + IntArrayTag worldTag = (IntArrayTag) location.get(CommonKeys.WORLD); + if (worldTag != null) { + worldUuid = UUIDSerializer.INSTANCE.deserialize(worldTag); + } + + BlockPosition position = new BlockPosition(null, location.getLong(CommonKeys.POSITION).getAsLong()); + + double x = position.getX(); + double y = position.getY(); + double z = position.getZ(); + float pitch = location.getFloat(CommonKeys.PITCH).orElse(0.0f); + float yaw = location.getFloat(CommonKeys.YAW).orElse(0.0f); + + return new Location( + worldUuid != null ? Bukkit.getWorld(worldUuid) : null, + x, y, z, yaw, pitch + ); + } +} diff --git a/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/UUIDSerializer.java b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/UUIDSerializer.java new file mode 100644 index 0000000000..7f1aba13a8 --- /dev/null +++ b/src/main/java/io/github/thebusybiscuit/slimefun4/storage/backend/binary/serializers/UUIDSerializer.java @@ -0,0 +1,43 @@ +package io.github.thebusybiscuit.slimefun4.storage.backend.binary.serializers; + +import java.util.UUID; + +import javax.annotation.Nonnull; + +import org.apache.commons.lang.Validate; + +import io.github.bakedlibs.dough.nbt.tags.IntArrayTag; + +public class UUIDSerializer implements BinarySerialize { + + public static final UUIDSerializer INSTANCE = new UUIDSerializer(); + + private UUIDSerializer() {} + + @Override + public IntArrayTag serialize(@Nonnull UUID uuid) { + Validate.notNull(uuid, "The provided UUID cannot be null"); + + long mostSig = uuid.getMostSignificantBits(); + long leastSig = uuid.getLeastSignificantBits(); + int[] ints = new int[] { + (int) (mostSig >> 32), + (int) mostSig, + (int) (leastSig >> 32), + (int) leastSig + }; + + return new IntArrayTag(ints); + } + + @Override + public UUID deserialize(@Nonnull IntArrayTag uuid) { + Validate.notNull(uuid, "The provided UUID cannot be null"); + + int[] ints = uuid.getValue(); + return new UUID( + (long) ints[0] << 32L | ints[1] & 0xFFFFFFFFL, + (long) ints[2] << 32L | ints[3] & 0xFFFFFFFFL + ); + } +} diff --git a/src/main/resources/config.yml b/src/main/resources/config.yml index cb133170e4..0673e99a4a 100644 --- a/src/main/resources/config.yml +++ b/src/main/resources/config.yml @@ -24,6 +24,9 @@ options: backup-data: true drop-block-creative: true +storage: + player-data: legacy + guide: show-vanilla-recipes: true receive-on-first-join: true