Registry sync packet compression II ()

* Optimize registry sync packet

* Make sure that the id map is sorted by the raw id
Send the difference of the first raw id and the last raw id from the bulk before

* Add packet size log prints

* Use Boolean.getBoolean
Don't send sync packet if the player is the server host

* Use generic Map

* Multiple raw id bulk test

* Group object namespace first before grouping raw ids

* Revert "Add packet size log prints"

This reverts commit fcb8b71e

* RegistrySyncPacket -> RegistryPacketSerializer
Move RegistryPacketSerializer singleton to the interface

* Packet size log prints

* Sort raw id on each namespace group

* Use Identifier.DEFAULT_NAMESPACE

* Slice to multiple packet if necessary

* Make MAX_PAYLOAD_SIZE configurable
Make testmod registry bigger
Print buffer size in equality test

* Move static packet handler instance to RegistrySyncManager
Release combinedBuf before discarding it
Discard syncedRegistryMap after it's accessed

* Use PacketByteBufs to create buffers
This commit is contained in:
deirn 2022-01-02 18:55:58 +00:00 committed by modmuss50
parent fb3b57b447
commit befed49ea3
13 changed files with 718 additions and 127 deletions

View file

@ -23,21 +23,23 @@ import net.minecraft.text.LiteralText;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking;
import net.fabricmc.fabric.impl.registry.sync.packet.RegistryPacketHandler;
public class FabricRegistryClientInit implements ClientModInitializer {
private static final Logger LOGGER = LogManager.getLogger();
@Override
public void onInitializeClient() {
ClientPlayNetworking.registerGlobalReceiver(RegistrySyncManager.ID, (client, handler, buf, responseSender) -> {
// if not hosting server, apply packet
RegistrySyncManager.receivePacket(client, buf, RegistrySyncManager.DEBUG || !client.isInSingleplayer(), (e) -> {
LOGGER.error("Registry remapping failed!", e);
registerSyncPacketReceiver(RegistrySyncManager.DIRECT_PACKET_HANDLER);
registerSyncPacketReceiver(RegistrySyncManager.NBT_PACKET_HANDLER);
}
client.execute(() -> {
handler.getConnection().disconnect(new LiteralText("Registry remapping failed: " + e.getMessage()));
});
});
});
private void registerSyncPacketReceiver(RegistryPacketHandler packetHandler) {
ClientPlayNetworking.registerGlobalReceiver(packetHandler.getPacketId(), (client, handler, buf, responseSender) ->
RegistrySyncManager.receivePacket(client, packetHandler, buf, RegistrySyncManager.DEBUG || !client.isInSingleplayer(), (e) -> {
LOGGER.error("Registry remapping failed!", e);
client.execute(() -> handler.getConnection().disconnect(new LiteralText("Registry remapping failed: " + e.getMessage())));
}));
}
}

View file

@ -21,10 +21,14 @@ import net.minecraft.util.registry.Registry;
import net.fabricmc.api.ModInitializer;
import net.fabricmc.fabric.api.event.registry.RegistryAttribute;
import net.fabricmc.fabric.api.event.registry.RegistryAttributeHolder;
import net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents;
public class FabricRegistryInit implements ModInitializer {
@Override
public void onInitialize() {
ServerPlayConnectionEvents.JOIN.register((handler, sender, server) ->
RegistrySyncManager.sendPacket(server, handler.player));
// Synced in PlaySoundS2CPacket.
RegistryAttributeHolder.get(Registry.SOUND_EVENT)
.addAttribute(RegistryAttribute.SYNCED);

View file

@ -0,0 +1,67 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.impl.registry.sync;
import java.util.LinkedHashMap;
import java.util.Map;
import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import net.minecraft.nbt.NbtCompound;
import net.minecraft.util.Identifier;
public class RegistryMapSerializer {
public static final int VERSION = 1;
public static Map<Identifier, Object2IntMap<Identifier>> fromNbt(NbtCompound nbt) {
NbtCompound mainNbt = nbt.getCompound("registries");
Map<Identifier, Object2IntMap<Identifier>> map = new LinkedHashMap<>();
for (String registryId : mainNbt.getKeys()) {
Object2IntMap<Identifier> idMap = new Object2IntLinkedOpenHashMap<>();
NbtCompound idNbt = mainNbt.getCompound(registryId);
for (String id : idNbt.getKeys()) {
idMap.put(new Identifier(id), idNbt.getInt(id));
}
map.put(new Identifier(registryId), idMap);
}
return map;
}
public static NbtCompound toNbt(Map<Identifier, Object2IntMap<Identifier>> map) {
NbtCompound mainNbt = new NbtCompound();
map.forEach((registryId, idMap) -> {
NbtCompound registryNbt = new NbtCompound();
for (Object2IntMap.Entry<Identifier> idPair : idMap.object2IntEntrySet()) {
registryNbt.putInt(idPair.getKey().toString(), idPair.getIntValue());
}
mainNbt.put(registryId.toString(), registryNbt);
});
NbtCompound nbt = new NbtCompound();
nbt.putInt("version", VERSION);
nbt.put("registries", mainNbt);
return nbt;
}
}

View file

@ -20,6 +20,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@ -28,9 +30,9 @@ import java.util.function.Consumer;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import io.netty.buffer.Unpooled;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import org.apache.logging.log4j.LogManager;
@ -38,55 +40,89 @@ import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import net.minecraft.nbt.NbtCompound;
import net.minecraft.network.Packet;
import net.minecraft.util.Identifier;
import net.minecraft.network.PacketByteBuf;
import net.minecraft.server.MinecraftServer;
import net.minecraft.server.network.ServerPlayerEntity;
import net.minecraft.util.Identifier;
import net.minecraft.util.registry.Registry;
import net.minecraft.util.thread.ThreadExecutor;
import net.fabricmc.fabric.api.event.registry.RegistryAttribute;
import net.fabricmc.fabric.api.event.registry.RegistryAttributeHolder;
import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking;
import net.fabricmc.fabric.impl.registry.sync.packet.DirectRegistryPacketHandler;
import net.fabricmc.fabric.impl.registry.sync.packet.NbtRegistryPacketHandler;
import net.fabricmc.fabric.impl.registry.sync.packet.RegistryPacketHandler;
public final class RegistrySyncManager {
static final boolean DEBUG = System.getProperty("fabric.registry.debug", "false").equalsIgnoreCase("true");
static final Identifier ID = new Identifier("fabric", "registry/sync");
public static final boolean DEBUG = Boolean.getBoolean("fabric.registry.debug");
@Deprecated
public static final RegistryPacketHandler NBT_PACKET_HANDLER = new NbtRegistryPacketHandler();
public static final RegistryPacketHandler DIRECT_PACKET_HANDLER = new DirectRegistryPacketHandler();
private static final Logger LOGGER = LogManager.getLogger("FabricRegistrySync");
private static final boolean DEBUG_WRITE_REGISTRY_DATA = System.getProperty("fabric.registry.debug.writeContentsAsCsv", "false").equalsIgnoreCase("true");
private static final boolean DEBUG_WRITE_REGISTRY_DATA = Boolean.getBoolean("fabric.registry.debug.writeContentsAsCsv");
private static final boolean FORCE_NBT_SYNC = Boolean.getBoolean("fabric.registry.forceNbtSync");
//Set to true after vanilla's bootstrap has completed
public static boolean postBootstrap = false;
private RegistrySyncManager() { }
public static Packet<?> createPacket() {
LOGGER.debug("Creating registry sync packet");
NbtCompound tag = toTag(true, null);
if (tag == null) {
return null;
public static void sendPacket(MinecraftServer server, ServerPlayerEntity player) {
if (!DEBUG && server.isHost(player.getGameProfile())) {
return;
}
PacketByteBuf buf = new PacketByteBuf(Unpooled.buffer());
buf.writeNbt(tag);
if (FORCE_NBT_SYNC) {
LOGGER.warn("Force NBT sync is enabled");
sendPacket(player, NBT_PACKET_HANDLER);
return;
}
return ServerPlayNetworking.createS2CPacket(ID, buf);
if (ServerPlayNetworking.canSend(player, DIRECT_PACKET_HANDLER.getPacketId())) {
sendPacket(player, DIRECT_PACKET_HANDLER);
} else {
LOGGER.warn("Player {} can't receive direct packet, using nbt packet instead", player.getEntityName());
sendPacket(player, NBT_PACKET_HANDLER);
}
}
public static void receivePacket(ThreadExecutor<?> executor, PacketByteBuf buf, boolean accept, Consumer<Exception> errorHandler) {
NbtCompound compound = buf.readNbt();
private static void sendPacket(ServerPlayerEntity player, RegistryPacketHandler handler) {
Map<Identifier, Object2IntMap<Identifier>> map = RegistrySyncManager.createAndPopulateRegistryMap(true, null);
if (map != null) {
handler.sendPacket(player, map);
}
}
public static void receivePacket(ThreadExecutor<?> executor, RegistryPacketHandler handler, PacketByteBuf buf, boolean accept, Consumer<Exception> errorHandler) {
handler.receivePacket(buf);
if (!handler.isPacketFinished()) {
return;
}
if (DEBUG) {
String handlerName = handler.getClass().getSimpleName();
LOGGER.info("{} total packet: {}", handlerName, handler.getTotalPacketReceived());
LOGGER.info("{} raw size: {}", handlerName, handler.getRawBufSize());
LOGGER.info("{} deflated size: {}", handlerName, handler.getDeflatedBufSize());
}
Map<Identifier, Object2IntMap<Identifier>> map = handler.getSyncedRegistryMap();
if (accept) {
try {
executor.submit(() -> {
if (compound == null) {
errorHandler.accept(new RemapException("Received null compound tag in sync packet!"));
if (map == null) {
errorHandler.accept(new RemapException("Received null map in sync packet!"));
return null;
}
try {
apply(compound, RemappableRegistry.RemapMode.REMOTE);
apply(map, RemappableRegistry.RemapMode.REMOTE);
} catch (RemapException e) {
errorHandler.accept(e);
}
@ -103,12 +139,12 @@ public final class RegistrySyncManager {
* Creates a {@link NbtCompound} used to save or sync the registry ids.
*
* @param isClientSync true when syncing to the client, false when saving
* @param activeTag contains the registry ids that were previously read and applied, can be null.
* @param activeMap contains the registry ids that were previously read and applied, can be null.
* @return a {@link NbtCompound} to save or sync, null when empty
*/
@Nullable
public static NbtCompound toTag(boolean isClientSync, @Nullable NbtCompound activeTag) {
NbtCompound mainTag = new NbtCompound();
public static Map<Identifier, Object2IntMap<Identifier>> createAndPopulateRegistryMap(boolean isClientSync, @Nullable Map<Identifier, Object2IntMap<Identifier>> activeMap) {
Map<Identifier, Object2IntMap<Identifier>> map = new LinkedHashMap<>();
for (Identifier registryId : Registry.REGISTRIES.getIds()) {
Registry registry = Registry.REGISTRIES.get(registryId);
@ -153,10 +189,10 @@ public final class RegistrySyncManager {
* This contains the previous state's registry data, this is used for a few things:
* Such as ensuring that previously modded registries or registry entries are not lost or overwritten.
*/
NbtCompound previousRegistryData = null;
Object2IntMap<Identifier> previousIdMap = null;
if (activeTag != null && activeTag.contains(registryId.toString())) {
previousRegistryData = activeTag.getCompound(registryId.toString());
if (activeMap != null && activeMap.containsKey(registryId)) {
previousIdMap = activeMap.get(registryId);
}
RegistryAttributeHolder attributeHolder = RegistryAttributeHolder.get(registry);
@ -174,10 +210,10 @@ public final class RegistrySyncManager {
* or a previous version of fabric registry sync, but will save these ids to disk in case the mod or mods
* are added back.
*/
if ((previousRegistryData == null || isClientSync) && !attributeHolder.hasAttribute(RegistryAttribute.MODDED)) {
if ((previousIdMap == null || isClientSync) && !attributeHolder.hasAttribute(RegistryAttribute.MODDED)) {
LOGGER.debug("Skipping un-modded registry: " + registryId);
continue;
} else if (previousRegistryData != null) {
} else if (previousIdMap != null) {
LOGGER.debug("Preserving previously modded registry: " + registryId);
}
@ -188,7 +224,7 @@ public final class RegistrySyncManager {
}
if (registry instanceof RemappableRegistry) {
NbtCompound registryTag = new NbtCompound();
Object2IntMap<Identifier> idMap = new Object2IntLinkedOpenHashMap<>();
IntSet rawIdsFound = DEBUG ? new IntOpenHashSet() : null;
for (Object o : registry) {
@ -213,57 +249,52 @@ public final class RegistrySyncManager {
}
}
registryTag.putInt(id.toString(), rawId);
idMap.put(id, rawId);
}
/*
* Look for existing registry key/values that are not in the current registries.
* This can happen when registry entries are removed, preventing that ID from being re-used by something else.
*/
if (!isClientSync && previousRegistryData != null) {
for (String key : previousRegistryData.getKeys()) {
if (!registryTag.contains(key)) {
if (!isClientSync && previousIdMap != null) {
for (Identifier key : previousIdMap.keySet()) {
if (!idMap.containsKey(key)) {
LOGGER.debug("Saving orphaned registry entry: " + key);
registryTag.putInt(key, previousRegistryData.getInt(key));
idMap.put(key, previousIdMap.getInt(key));
}
}
}
mainTag.put(registryId.toString(), registryTag);
map.put(registryId, idMap);
}
}
// Ensure any orphaned registry's are kept on disk
if (!isClientSync && activeTag != null) {
for (String registryKey : activeTag.getKeys()) {
if (!mainTag.contains(registryKey)) {
if (!isClientSync && activeMap != null) {
for (Identifier registryKey : activeMap.keySet()) {
if (!map.containsKey(registryKey)) {
LOGGER.debug("Saving orphaned registry: " + registryKey);
mainTag.put(registryKey, activeTag.getCompound(registryKey));
map.put(registryKey, activeMap.get(registryKey));
}
}
}
if (mainTag.getKeys().isEmpty()) {
if (map.isEmpty()) {
return null;
}
NbtCompound tag = new NbtCompound();
tag.putInt("version", 1);
tag.put("registries", mainTag);
return tag;
return map;
}
public static NbtCompound apply(NbtCompound tag, RemappableRegistry.RemapMode mode) throws RemapException {
NbtCompound mainTag = tag.getCompound("registries");
Set<String> containedRegistries = Sets.newHashSet(mainTag.getKeys());
public static void apply(Map<Identifier, Object2IntMap<Identifier>> map, RemappableRegistry.RemapMode mode) throws RemapException {
Set<Identifier> containedRegistries = Sets.newHashSet(map.keySet());
for (Identifier registryId : Registry.REGISTRIES.getIds()) {
if (!containedRegistries.remove(registryId.toString())) {
if (!containedRegistries.remove(registryId)) {
continue;
}
NbtCompound registryTag = mainTag.getCompound(registryId.toString());
Object2IntMap<Identifier> registryMap = map.get(registryId);
Registry registry = Registry.REGISTRIES.get(registryId);
RegistryAttributeHolder attributeHolder = RegistryAttributeHolder.get(registry);
@ -276,8 +307,8 @@ public final class RegistrySyncManager {
if (registry instanceof RemappableRegistry) {
Object2IntMap<Identifier> idMap = new Object2IntOpenHashMap<>();
for (String key : registryTag.getKeys()) {
idMap.put(new Identifier(key), registryTag.getInt(key));
for (Identifier key : registryMap.keySet()) {
idMap.put(key, registryMap.getInt(key));
}
((RemappableRegistry) registry).remap(registryId.toString(), idMap, mode);
@ -287,8 +318,6 @@ public final class RegistrySyncManager {
if (!containedRegistries.isEmpty()) {
LOGGER.warn("[fabric-registry-sync] Could not find the following registries: " + Joiner.on(", ").join(containedRegistries));
}
return mainTag;
}
public static void unmap() throws RemapException {

View file

@ -0,0 +1,247 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.impl.registry.sync.packet;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.jetbrains.annotations.Nullable;
import net.minecraft.network.PacketByteBuf;
import net.minecraft.server.network.ServerPlayerEntity;
import net.minecraft.util.Identifier;
import net.fabricmc.fabric.api.networking.v1.PacketByteBufs;
/**
* A more optimized method to sync registry ids to client.
* Produce smaller packet than old {@link NbtRegistryPacketHandler nbt-based} method.
*
* <p>This method optimize the packet in multiple way:
* <ul>
* <li>Directly write into the buffer instead of using an nbt;</li>
* <li>Group all {@link Identifier} with same namespace together and only send those unique namespaces once for each group;</li>
* <li>Group consecutive rawIds together and only send the difference of the first rawId and the last rawId of the bulk before.
* This is based on the assumption that mods generally register all of their object at once,
* therefore making the rawIds somewhat densely packed.</li>
* </ul>
*
* <p>This method also split into multiple packets if it exceeds the limit, defaults to 1 MB.
*/
public class DirectRegistryPacketHandler extends RegistryPacketHandler {
/**
* @see net.minecraft.network.packet.s2c.play.CustomPayloadS2CPacket#MAX_PAYLOAD_SIZE
*/
@SuppressWarnings("JavadocReference")
private static final int MAX_PAYLOAD_SIZE = Integer.getInteger("fabric.registry.direct.maxPayloadSize", 0x100000);
private static final Identifier ID = new Identifier("fabric", "registry/sync/direct");
@Nullable
private PacketByteBuf combinedBuf;
@Nullable
private Map<Identifier, Object2IntMap<Identifier>> syncedRegistryMap;
private boolean isPacketFinished = false;
private int totalPacketReceived = 0;
@Override
public Identifier getPacketId() {
return ID;
}
@Override
public void sendPacket(ServerPlayerEntity player, Map<Identifier, Object2IntMap<Identifier>> registryMap) {
PacketByteBuf buf = PacketByteBufs.create();
// Group registry ids with same namespace.
Map<String, List<Identifier>> regNamespaceGroups = registryMap.keySet().stream()
.collect(Collectors.groupingBy(Identifier::getNamespace));
buf.writeVarInt(regNamespaceGroups.size());
regNamespaceGroups.forEach((regNamespace, regIds) -> {
buf.writeString(optimizeNamespace(regNamespace));
buf.writeVarInt(regIds.size());
for (Identifier regId : regIds) {
buf.writeString(regId.getPath());
Object2IntMap<Identifier> idMap = registryMap.get(regId);
// Sort object ids by its namespace. We use linked map here to keep the original namespace ordering.
Map<String, List<Object2IntMap.Entry<Identifier>>> idNamespaceGroups = idMap.object2IntEntrySet().stream()
.collect(Collectors.groupingBy(e -> e.getKey().getNamespace(), LinkedHashMap::new, Collectors.toCollection(ArrayList::new)));
buf.writeVarInt(idNamespaceGroups.size());
int lastBulkLastRawId = 0;
for (Map.Entry<String, List<Object2IntMap.Entry<Identifier>>> idNamespaceEntry : idNamespaceGroups.entrySet()) {
// Make sure the ids are sorted by its raw id.
List<Object2IntMap.Entry<Identifier>> idPairs = idNamespaceEntry.getValue();
idPairs.sort(Comparator.comparingInt(Object2IntMap.Entry::getIntValue));
// Group consecutive raw ids together.
List<List<Object2IntMap.Entry<Identifier>>> bulks = new ArrayList<>();
Iterator<Object2IntMap.Entry<Identifier>> idPairIter = idPairs.iterator();
List<Object2IntMap.Entry<Identifier>> currentBulk = new ArrayList<>();
Object2IntMap.Entry<Identifier> currentPair = idPairIter.next();
currentBulk.add(currentPair);
while (idPairIter.hasNext()) {
currentPair = idPairIter.next();
if (currentBulk.get(currentBulk.size() - 1).getIntValue() + 1 != currentPair.getIntValue()) {
bulks.add(currentBulk);
currentBulk = new ArrayList<>();
}
currentBulk.add(currentPair);
}
bulks.add(currentBulk);
buf.writeString(optimizeNamespace(idNamespaceEntry.getKey()));
buf.writeVarInt(bulks.size());
for (List<Object2IntMap.Entry<Identifier>> bulk : bulks) {
int firstRawId = bulk.get(0).getIntValue();
int bulkRawIdStartDiff = firstRawId - lastBulkLastRawId;
buf.writeVarInt(bulkRawIdStartDiff);
buf.writeVarInt(bulk.size());
for (Object2IntMap.Entry<Identifier> idPair : bulk) {
buf.writeString(idPair.getKey().getPath());
lastBulkLastRawId = idPair.getIntValue();
}
}
}
}
});
// Split the packet to multiple MAX_PAYLOAD_SIZEd buffers.
int readableBytes = buf.readableBytes();
int sliceIndex = 0;
while (sliceIndex < readableBytes) {
int sliceSize = Math.min(readableBytes - sliceIndex, MAX_PAYLOAD_SIZE);
PacketByteBuf slicedBuf = PacketByteBufs.slice(buf, sliceIndex, sliceSize);
sendPacket(player, slicedBuf);
sliceIndex += sliceSize;
}
// Send an empty buffer to mark the end of the split.
sendPacket(player, PacketByteBufs.empty());
}
@Override
public void receivePacket(PacketByteBuf slicedBuf) {
Preconditions.checkState(!isPacketFinished);
totalPacketReceived++;
if (combinedBuf == null) {
combinedBuf = PacketByteBufs.create();
}
if (slicedBuf.readableBytes() != 0) {
combinedBuf.writeBytes(slicedBuf);
return;
}
isPacketFinished = true;
computeBufSize(combinedBuf);
syncedRegistryMap = new LinkedHashMap<>();
int regNamespaceGroupAmount = combinedBuf.readVarInt();
for (int i = 0; i < regNamespaceGroupAmount; i++) {
String regNamespace = combinedBuf.readString();
int regNamespaceGroupLength = combinedBuf.readVarInt();
for (int j = 0; j < regNamespaceGroupLength; j++) {
String regPath = combinedBuf.readString();
Object2IntMap<Identifier> idMap = new Object2IntLinkedOpenHashMap<>();
int idNamespaceGroupAmount = combinedBuf.readVarInt();
int lastBulkLastRawId = 0;
for (int k = 0; k < idNamespaceGroupAmount; k++) {
String idNamespace = combinedBuf.readString();
int rawIdBulkAmount = combinedBuf.readVarInt();
for (int l = 0; l < rawIdBulkAmount; l++) {
int bulkRawIdStartDiff = combinedBuf.readVarInt();
int bulkSize = combinedBuf.readVarInt();
int currentRawId = (lastBulkLastRawId + bulkRawIdStartDiff) - 1;
for (int m = 0; m < bulkSize; m++) {
currentRawId++;
String idPath = combinedBuf.readString();
idMap.put(new Identifier(idNamespace, idPath), currentRawId);
}
lastBulkLastRawId = currentRawId;
}
}
syncedRegistryMap.put(new Identifier(regNamespace, regPath), idMap);
}
}
combinedBuf.release();
combinedBuf = null;
}
@Override
public boolean isPacketFinished() {
return isPacketFinished;
}
@Override
public int getTotalPacketReceived() {
Preconditions.checkState(isPacketFinished);
return totalPacketReceived;
}
@Override
@Nullable
public Map<Identifier, Object2IntMap<Identifier>> getSyncedRegistryMap() {
Preconditions.checkState(isPacketFinished);
Map<Identifier, Object2IntMap<Identifier>> map = syncedRegistryMap;
isPacketFinished = false;
totalPacketReceived = 0;
syncedRegistryMap = null;
return map;
}
private String optimizeNamespace(String namespace) {
return namespace.equals(Identifier.DEFAULT_NAMESPACE) ? "" : namespace;
}
}

View file

@ -0,0 +1,77 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.impl.registry.sync.packet;
import java.util.Map;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.jetbrains.annotations.Nullable;
import net.minecraft.nbt.NbtCompound;
import net.minecraft.network.PacketByteBuf;
import net.minecraft.server.network.ServerPlayerEntity;
import net.minecraft.util.Identifier;
import net.fabricmc.fabric.api.networking.v1.PacketByteBufs;
import net.fabricmc.fabric.impl.registry.sync.RegistryMapSerializer;
/**
* A method to sync registry ids using {@link NbtCompound} and {@link PacketByteBuf#writeNbt}.
* Kept here for old version support.
*/
// TODO: Remove
@Deprecated
public class NbtRegistryPacketHandler extends RegistryPacketHandler {
private static final Identifier ID = new Identifier("fabric", "registry/sync");
private Map<Identifier, Object2IntMap<Identifier>> syncedRegistryMap;
@Override
public Identifier getPacketId() {
return ID;
}
@Override
public void sendPacket(ServerPlayerEntity player, Map<Identifier, Object2IntMap<Identifier>> registryMap) {
PacketByteBuf buf = PacketByteBufs.create();
buf.writeNbt(RegistryMapSerializer.toNbt(registryMap));
sendPacket(player, buf);
}
@Override
public void receivePacket(PacketByteBuf buf) {
computeBufSize(buf);
NbtCompound nbt = buf.readNbt();
syncedRegistryMap = nbt != null ? RegistryMapSerializer.fromNbt(nbt) : null;
}
@Override
public int getTotalPacketReceived() {
return 1;
}
@Override
public boolean isPacketFinished() {
return true;
}
@Override
@Nullable
public Map<Identifier, Object2IntMap<Identifier>> getSyncedRegistryMap() {
return syncedRegistryMap;
}
}

View file

@ -0,0 +1,96 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.impl.registry.sync.packet;
import java.util.Map;
import java.util.zip.Deflater;
import io.netty.buffer.ByteBuf;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.jetbrains.annotations.Nullable;
import net.minecraft.network.PacketByteBuf;
import net.minecraft.server.network.ServerPlayerEntity;
import net.minecraft.util.Identifier;
import net.fabricmc.fabric.api.networking.v1.PacketByteBufs;
import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking;
import net.fabricmc.fabric.impl.registry.sync.RegistrySyncManager;
public abstract class RegistryPacketHandler {
private int rawBufSize = 0;
private int deflatedBufSize = 0;
public abstract Identifier getPacketId();
public abstract void sendPacket(ServerPlayerEntity player, Map<Identifier, Object2IntMap<Identifier>> registryMap);
public abstract void receivePacket(PacketByteBuf buf);
public abstract int getTotalPacketReceived();
public abstract boolean isPacketFinished();
@Nullable
public abstract Map<Identifier, Object2IntMap<Identifier>> getSyncedRegistryMap();
protected final void sendPacket(ServerPlayerEntity player, PacketByteBuf buf) {
ServerPlayNetworking.send(player, getPacketId(), buf);
}
protected final void computeBufSize(PacketByteBuf buf) {
if (!RegistrySyncManager.DEBUG) {
return;
}
final byte[] deflateBuffer = new byte[8192];
ByteBuf byteBuf = buf.copy();
Deflater deflater = new Deflater();
int i = byteBuf.readableBytes();
PacketByteBuf deflatedBuf = PacketByteBufs.create();
if (i < 256) {
deflatedBuf.writeVarInt(0);
deflatedBuf.writeBytes(byteBuf);
} else {
byte[] bs = new byte[i];
byteBuf.readBytes(bs);
deflatedBuf.writeVarInt(bs.length);
deflater.setInput(bs, 0, i);
deflater.finish();
while (!deflater.finished()) {
int j = deflater.deflate(deflateBuffer);
deflatedBuf.writeBytes(deflateBuffer, 0, j);
}
deflater.reset();
}
rawBufSize = buf.readableBytes();
deflatedBufSize = deflatedBuf.readableBytes();
}
public final int getRawBufSize() {
return rawBufSize;
}
public final int getDeflatedBufSize() {
return deflatedBufSize;
}
}

View file

@ -23,7 +23,9 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.spongepowered.asm.mixin.Final;
@ -37,10 +39,12 @@ import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
import net.minecraft.nbt.NbtCompound;
import net.minecraft.nbt.NbtIo;
import net.minecraft.util.Identifier;
import net.minecraft.world.SaveProperties;
import net.minecraft.world.level.storage.LevelStorage;
import net.minecraft.util.registry.DynamicRegistryManager;
import net.fabricmc.fabric.impl.registry.sync.RegistryMapSerializer;
import net.fabricmc.fabric.impl.registry.sync.RegistrySyncManager;
import net.fabricmc.fabric.impl.registry.sync.RemapException;
import net.fabricmc.fabric.impl.registry.sync.RemappableRegistry;
@ -50,11 +54,11 @@ public class MixinLevelStorageSession {
@Unique
private static final int FABRIC_ID_REGISTRY_BACKUPS = 3;
@Unique
private static Logger FABRIC_LOGGER = LogManager.getLogger("FabricRegistrySync");
private static final Logger FABRIC_LOGGER = LogManager.getLogger("FabricRegistrySync");
@Unique
private NbtCompound fabric_lastSavedIdMap = null;
private Map<Identifier, Object2IntMap<Identifier>> fabric_lastSavedRegistryMap = null;
@Unique
private NbtCompound fabric_activeTag = null;
private Map<Identifier, Object2IntMap<Identifier>> fabric_activeRegistryMap = null;
@Shadow
@Final
@ -70,7 +74,8 @@ public class MixinLevelStorageSession {
fileInputStream.close();
if (tag != null) {
fabric_activeTag = RegistrySyncManager.apply(tag, RemappableRegistry.RemapMode.AUTHORITATIVE);
fabric_activeRegistryMap = RegistryMapSerializer.fromNbt(tag);
RegistrySyncManager.apply(fabric_activeRegistryMap, RemappableRegistry.RemapMode.AUTHORITATIVE);
return true;
}
}
@ -86,14 +91,14 @@ public class MixinLevelStorageSession {
@Unique
private void fabric_saveRegistryData() {
FABRIC_LOGGER.debug("Starting registry save");
NbtCompound newIdMap = RegistrySyncManager.toTag(false, fabric_activeTag);
Map<Identifier, Object2IntMap<Identifier>> newMap = RegistrySyncManager.createAndPopulateRegistryMap(false, fabric_activeRegistryMap);
if (newIdMap == null) {
if (newMap == null) {
FABRIC_LOGGER.debug("Not saving empty registry data");
return;
}
if (!newIdMap.equals(fabric_lastSavedIdMap)) {
if (!newMap.equals(fabric_lastSavedRegistryMap)) {
for (int i = FABRIC_ID_REGISTRY_BACKUPS - 1; i >= 0; i--) {
File file = fabric_getWorldIdMapFile(i);
@ -117,15 +122,15 @@ public class MixinLevelStorageSession {
}
}
FABRIC_LOGGER.debug("Saving registry data to " + file.toString());
FABRIC_LOGGER.debug("Saving registry data to " + file);
FileOutputStream fileOutputStream = new FileOutputStream(file);
NbtIo.writeCompressed(newIdMap, fileOutputStream);
NbtIo.writeCompressed(RegistryMapSerializer.toNbt(newMap), fileOutputStream);
fileOutputStream.close();
} catch (IOException e) {
FABRIC_LOGGER.warn("[fabric-registry-sync] Failed to save registry file!", e);
}
fabric_lastSavedIdMap = newIdMap;
fabric_lastSavedRegistryMap = newMap;
}
}

View file

@ -1,43 +0,0 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.mixin.registry.sync;
import org.spongepowered.asm.mixin.Mixin;
import org.spongepowered.asm.mixin.injection.At;
import org.spongepowered.asm.mixin.injection.Inject;
import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
import net.minecraft.network.ClientConnection;
import net.minecraft.network.Packet;
import net.minecraft.server.PlayerManager;
import net.minecraft.server.network.ServerPlayerEntity;
import net.fabricmc.fabric.impl.registry.sync.RegistrySyncManager;
@Mixin(PlayerManager.class)
public abstract class MixinPlayerManager {
@Inject(method = "onPlayerConnect", at = @At(value = "INVOKE", target = "Lnet/minecraft/network/packet/s2c/play/DifficultyS2CPacket;<init>(Lnet/minecraft/world/Difficulty;Z)V"))
public void onPlayerConnect(ClientConnection lvt1, ServerPlayerEntity lvt2, CallbackInfo info) {
// TODO: If integrated and local, don't send the packet (it's ignored)
// TODO: Refactor out into network + move registry hook to event
Packet<?> packet = RegistrySyncManager.createPacket();
if (packet != null) {
lvt2.networkHandler.sendPacket(packet);
}
}
}

View file

@ -9,7 +9,6 @@
"MixinDynamicRegistryManager",
"MixinIdList",
"MixinIdRegistry",
"MixinPlayerManager",
"MixinLevelStorageSession",
"MixinRegistry",
"MixinSimpleRegistry",

View file

@ -16,6 +16,9 @@
package net.fabricmc.fabric.test.registry.sync;
import java.util.Map;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.apache.commons.lang3.Validate;
import net.minecraft.block.AbstractBlock;
@ -38,6 +41,12 @@ import net.fabricmc.fabric.api.event.registry.FabricRegistryBuilder;
import net.fabricmc.fabric.api.event.registry.RegistryAttribute;
import net.fabricmc.fabric.api.event.registry.RegistryAttributeHolder;
import net.fabricmc.fabric.api.event.registry.RegistryEntryAddedCallback;
import net.fabricmc.fabric.api.networking.v1.PacketByteBufs;
import net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents;
import net.fabricmc.fabric.impl.registry.sync.RegistrySyncManager;
import net.fabricmc.fabric.impl.registry.sync.packet.DirectRegistryPacketHandler;
import net.fabricmc.fabric.impl.registry.sync.packet.NbtRegistryPacketHandler;
import net.fabricmc.fabric.impl.registry.sync.packet.RegistryPacketHandler;
public class RegistrySyncTest implements ModInitializer {
/**
@ -46,20 +55,40 @@ public class RegistrySyncTest implements ModInitializer {
public static final boolean REGISTER_BLOCKS = Boolean.parseBoolean(System.getProperty("fabric.registry.sync.test.register.blocks", "true"));
public static final boolean REGISTER_ITEMS = Boolean.parseBoolean(System.getProperty("fabric.registry.sync.test.register.items", "true"));
public static final Identifier PACKET_CHECK_DIRECT = new Identifier("fabric-registry-sync-v0-v1-testmod:packet_check/direct");
public static final RegistryPacketHandler DIRECT_PACKET_HANDLER = new DirectRegistryPacketHandler() {
@Override
public Identifier getPacketId() {
return PACKET_CHECK_DIRECT;
}
};
public static final Identifier PACKET_CHECK_NBT = new Identifier("fabric-registry-sync-v0-v1-testmod:packet_check/nbt");
public static final RegistryPacketHandler NBT_PACKET_HANDLER = new NbtRegistryPacketHandler() {
@Override
public Identifier getPacketId() {
return PACKET_CHECK_NBT;
}
};
public static final Identifier PACKET_CHECK_COMPARE = new Identifier("fabric-registry-sync-v0-v1-testmod:packet_check/compare");
@Override
public void onInitialize() {
ServerPlayConnectionEvents.JOIN.register((handler, sender, server) -> {
Map<Identifier, Object2IntMap<Identifier>> map = RegistrySyncManager.createAndPopulateRegistryMap(true, null);
NBT_PACKET_HANDLER.sendPacket(handler.player, map);
DIRECT_PACKET_HANDLER.sendPacket(handler.player, map);
sender.sendPacket(PACKET_CHECK_COMPARE, PacketByteBufs.empty());
});
testBuiltInRegistrySync();
if (REGISTER_BLOCKS) {
for (int i = 0; i < 5; i++) {
Block block = new Block(AbstractBlock.Settings.of(Material.STONE));
Registry.register(Registry.BLOCK, new Identifier("registry_sync", "block_" + i), block);
if (REGISTER_ITEMS) {
BlockItem blockItem = new BlockItem(block, new Item.Settings());
Registry.register(Registry.ITEM, new Identifier("registry_sync", "block_" + i), blockItem);
}
}
// For checking raw id bulk in direct registry packet, make registry_sync namespace have two bulks.
registerBlocks("registry_sync", 5, 0);
registerBlocks("registry_sync2", 50, 0);
registerBlocks("registry_sync", 2, 5);
Validate.isTrue(RegistryAttributeHolder.get(Registry.BLOCK).hasAttribute(RegistryAttribute.MODDED), "Modded block was registered but registry not marked as modded");
@ -87,6 +116,18 @@ public class RegistrySyncTest implements ModInitializer {
});
}
private static void registerBlocks(String namespace, int amount, int startingId) {
for (int i = 0; i < amount; i++) {
Block block = new Block(AbstractBlock.Settings.of(Material.STONE));
Registry.register(Registry.BLOCK, new Identifier(namespace, "block_" + (i + startingId)), block);
if (REGISTER_ITEMS) {
BlockItem blockItem = new BlockItem(block, new Item.Settings());
Registry.register(Registry.ITEM, new Identifier(namespace, "block_" + (i + startingId)), blockItem);
}
}
}
/**
* Tests that built-in registries are properly synchronized even after the dynamic reigstry managers have been
* class-loaded.

View file

@ -0,0 +1,64 @@
/*
* Copyright (c) 2016, 2017, 2018, 2019 FabricMC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.fabricmc.fabric.test.registry.sync;
import java.util.Map;
import java.util.Objects;
import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import net.minecraft.util.Identifier;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.api.EnvType;
import net.fabricmc.api.Environment;
import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking;
import net.fabricmc.fabric.impl.registry.sync.packet.RegistryPacketHandler;
@Environment(EnvType.CLIENT)
public class RegistrySyncTestClient implements ClientModInitializer {
private static final Logger LOGGER = LogManager.getLogger();
@Override
public void onInitializeClient() {
ClientPlayNetworking.registerGlobalReceiver(RegistrySyncTest.PACKET_CHECK_DIRECT, (client, handler, buf, responseSender) ->
RegistrySyncTest.DIRECT_PACKET_HANDLER.receivePacket(buf));
ClientPlayNetworking.registerGlobalReceiver(RegistrySyncTest.PACKET_CHECK_NBT, (client, handler, buf, responseSender) ->
RegistrySyncTest.NBT_PACKET_HANDLER.receivePacket(buf));
ClientPlayNetworking.registerGlobalReceiver(RegistrySyncTest.PACKET_CHECK_COMPARE, (client, handler, buf, responseSender) -> {
logBufferSize(RegistrySyncTest.NBT_PACKET_HANDLER);
logBufferSize(RegistrySyncTest.DIRECT_PACKET_HANDLER);
Map<Identifier, Object2IntMap<Identifier>> directPacketMap = RegistrySyncTest.DIRECT_PACKET_HANDLER.getSyncedRegistryMap();
Map<Identifier, Object2IntMap<Identifier>> nbtPacketMap = RegistrySyncTest.NBT_PACKET_HANDLER.getSyncedRegistryMap();
Preconditions.checkArgument(Objects.requireNonNull(nbtPacketMap).equals(directPacketMap), "nbt packet and direct packet are not equal!");
});
}
private void logBufferSize(RegistryPacketHandler handler) {
String handlerName = handler.getClass().getSuperclass().getSimpleName();
LOGGER.info("{} total packet: {}", handlerName, handler.getTotalPacketReceived());
LOGGER.info("{} raw size: {}", handlerName, handler.getRawBufSize());
LOGGER.info("{} deflated size: {}", handlerName, handler.getDeflatedBufSize());
}
}

View file

@ -11,6 +11,9 @@
"entrypoints": {
"main": [
"net.fabricmc.fabric.test.registry.sync.RegistrySyncTest"
],
"client": [
"net.fabricmc.fabric.test.registry.sync.RegistrySyncTestClient"
]
}
}