Skip to content

Commit

Permalink
Work on default extraction system
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukebemish
lukebemish committed Jan 31, 2024
1 parent c3bbba5 commit 29f60de
Show file tree
Hide file tree
Showing 7 changed files with 192 additions and 43 deletions.
15 changes: 15 additions & 0 deletions common/src/main/java/dev/lukebemish/defaultresources/api/OutdatedResourcesListener.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package dev.lukebemish.defaultresources.api;

import dev.lukebemish.defaultresources.impl.DefaultResources;

public interface OutdatedResourcesListener {
void resourcesOutdated();

static void register(String modId, OutdatedResourcesListener listener) {
if (DefaultResources.isTargetOutdated(modId)) {
listener.resourcesOutdated();
} else {
DefaultResources.addListener(modId, listener);
}
}
}
12 changes: 7 additions & 5 deletions common/src/main/java/dev/lukebemish/defaultresources/impl/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,21 @@
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.zip.ZipFile;

public record Config(HashMap<String, ExtractionState> extract, HashMap<String, Boolean> fromResourcePacks) {
public record Config(ConcurrentHashMap<String, ExtractionState> extract, HashMap<String, Boolean> fromResourcePacks) {
public static final Codec<Config> CODEC = RecordCodecBuilder.create(i -> i.group(
Codec.unboundedMap(Codec.STRING, StringRepresentable.fromEnum(ExtractionState::values)).xmap(HashMap::new, Function.identity()).fieldOf("extract").forGetter(Config::extract),
Codec.unboundedMap(Codec.STRING, StringRepresentable.fromEnum(ExtractionState::values)).xmap(ConcurrentHashMap::new, Function.identity()).fieldOf("extract").forGetter(Config::extract),
Codec.unboundedMap(Codec.STRING, Codec.BOOL).fieldOf("from_resource_packs").xmap(HashMap::new, Function.identity()).forGetter(Config::fromResourcePacks)
).apply(i, Config::new));

public static final Supplier<Config> INSTANCE = Suppliers.memoize(Config::readFromConfig);

private static Config getDefault() {
return new Config(new HashMap<>(), new HashMap<>());
return new Config(new ConcurrentHashMap<>(), new HashMap<>());
}

private static Config readFromConfig() {
Expand All @@ -53,7 +54,7 @@ private static Config readFromConfig() {
// Already caught and logged.
}
}
var map = new HashMap<>(config.extract());
var map = new ConcurrentHashMap<>(config.extract());
Services.PLATFORM.getExistingModdedPaths(DefaultResources.META_FILE_PATH).forEach((modId, metaPath) -> {
if (!map.containsKey(modId)) {
map.put(modId, ExtractionState.UNEXTRACTED);
Expand Down Expand Up @@ -156,7 +157,8 @@ public void save() {
enum ExtractionState implements StringRepresentable {
UNEXTRACTED,
EXTRACT,
EXTRACTED;
EXTRACTED,
OUTDATED;

@Override
public @NonNull String getSerializedName() {
Expand Down
194 changes: 160 additions & 34 deletions common/src/main/java/dev/lukebemish/defaultresources/impl/DefaultResources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,15 @@
import com.mojang.datafixers.util.Pair;
import com.mojang.serialization.JsonOps;
import dev.lukebemish.defaultresources.api.GlobalResourceManager;
import dev.lukebemish.defaultresources.api.OutdatedResourcesListener;
import net.minecraft.server.packs.PackResources;
import net.minecraft.server.packs.PackType;
import net.minecraft.server.packs.repository.Pack;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jspecify.annotations.NonNull;
import org.jspecify.annotations.Nullable;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -28,70 +31,190 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.zip.Adler32;
import java.util.zip.Checksum;

public class DefaultResources {
public static final String MOD_ID = "defaultresources";
public static final Logger LOGGER = LogManager.getLogger(MOD_ID);

private static final int BUFFER_SIZE = 1024;
private static final Set<String> OUTDATED_TARGETS = new HashSet<>();
private volatile static boolean GLOBAL_SETUP = false;
private static final Map<String, List<OutdatedResourcesListener>> OUTDATED_RESOURCES_LISTENERS = new ConcurrentHashMap<>();
public static final String META_FILE_PATH = DefaultResources.MOD_ID + ".meta.json";
public static final String CHECK_FILE_PATH = DefaultResources.MOD_ID + ".checksum";

public static final Gson GSON = new GsonBuilder().setLenient().setPrettyPrinting().create();

private static final Map<String, BiFunction<String, PackType, Supplier<PackResources>>> QUEUED_RESOURCES = new HashMap<>();
private static final Map<String, BiFunction<String, PackType, Supplier<PackResources>>> QUEUED_STATIC_RESOURCES = new HashMap<>();
private static final Map<String, BiFunction<String, PackType, Supplier<PackResources>>> QUEUED_RESOURCES = new ConcurrentHashMap<>();
private static final Map<String, BiFunction<String, PackType, Supplier<PackResources>>> QUEUED_STATIC_RESOURCES = new ConcurrentHashMap<>();
public static final String GLOBAL_PREFIX = "global";

public static final GlobalResourceManager STATIC_ASSETS = createStaticResourceManager(PackType.CLIENT_RESOURCES);
public static final GlobalResourceManager STATIC_DATA = createStaticResourceManager(PackType.SERVER_DATA);

public static void addListener(String modId, OutdatedResourcesListener listener) {
OUTDATED_RESOURCES_LISTENERS.computeIfAbsent(modId, s -> new ArrayList<>()).add(listener);
}

public static void forMod(Function<String, Path> inJarPathGetter, String modId) {
Path defaultResourcesMeta = inJarPathGetter.apply(META_FILE_PATH);
ModMetaFile meta;
if (Files.exists(defaultResourcesMeta)) {
try (InputStream is = Files.newInputStream(defaultResourcesMeta)) {
JsonObject obj = GSON.fromJson(new BufferedReader(new InputStreamReader(is)), JsonObject.class);
ModMetaFile meta = ModMetaFile.CODEC.parse(JsonOps.INSTANCE, obj).getOrThrow(false, e -> {
});
Path defaultResources = inJarPathGetter.apply(meta.resourcesPath());

if (Files.exists(defaultResources)) {
Config.ExtractionState extractionState = Config.INSTANCE.get().extract().getOrDefault(modId, Config.ExtractionState.UNEXTRACTED);
if (extractionState == Config.ExtractionState.UNEXTRACTED) {
QUEUED_RESOURCES.put("__unextracted_" + modId, (s, type) -> {
if (!Files.exists(defaultResources.resolve(type.getDirectory()))) return null;
return () -> new AutoMetadataPathPackResources(s, "", defaultResources, type);
});
QUEUED_STATIC_RESOURCES.put("__unextracted_" + modId, (s, type) -> {
if (!Files.exists(defaultResources.resolve(GLOBAL_PREFIX+type.getDirectory()))) return null;
return () -> new AutoMetadataPathPackResources(s, GLOBAL_PREFIX, defaultResources, type);
});
} else if (extractionState == Config.ExtractionState.EXTRACT) {
Config.INSTANCE.get().extract().put(modId, Config.ExtractionState.EXTRACTED);
if (!meta.zip()) {
Path outPath = Services.PLATFORM.getGlobalFolder().resolve(modId);
if (!Files.exists(outPath))
copyResources(defaultResources, outPath);
meta = ModMetaFile.CODEC.parse(JsonOps.INSTANCE, obj).getOrThrow(false, e -> {});
} catch (IOException | RuntimeException e) {
DefaultResources.LOGGER.error("Could not read meta file for mod {}", modId, e);
return;
}
} else {
try {
meta = ModMetaFile.CODEC.parse(JsonOps.INSTANCE, new JsonObject()).getOrThrow(false, e -> {});
} catch (RuntimeException e) {
DefaultResources.LOGGER.error("Could not parse default meta file", e);
return;
}
}
Path defaultResources = inJarPathGetter.apply(meta.resourcesPath());

try {
if (Files.exists(defaultResources)) {
var defaultExtraction = meta.extract() ? Config.ExtractionState.EXTRACT : Config.ExtractionState.UNEXTRACTED;
Config.ExtractionState extractionState = Config.INSTANCE.get().extract().getOrDefault(modId, defaultExtraction);
if (!Config.INSTANCE.get().extract().containsKey(modId)) {
Config.INSTANCE.get().extract().put(modId, defaultExtraction);
}
if (extractionState == Config.ExtractionState.UNEXTRACTED) {
QUEUED_RESOURCES.put("__unextracted_" + modId, (s, type) -> {
if (!Files.exists(defaultResources.resolve(type.getDirectory()))) return null;
return () -> new AutoMetadataPathPackResources(s, "", defaultResources, type);
});
QUEUED_STATIC_RESOURCES.put("__unextracted_" + modId, (s, type) -> {
if (!Files.exists(defaultResources.resolve(GLOBAL_PREFIX + type.getDirectory()))) return null;
return () -> new AutoMetadataPathPackResources(s, GLOBAL_PREFIX, defaultResources, type);
});
} else if (extractionState == Config.ExtractionState.EXTRACT) {
Config.INSTANCE.get().extract().put(modId, meta.extract() ? Config.ExtractionState.EXTRACT : Config.ExtractionState.EXTRACTED);
if (!meta.zip()) {
Path outPath = Services.PLATFORM.getGlobalFolder().resolve(modId);
String checksum = shouldCopy(defaultResources, outPath, Files.exists(outPath));
if (checksum != null) {
copyResources(defaultResources, outPath, checksum);
} else {
DefaultResources.LOGGER.error("Could not extract default resources for mod {} because they are already extracted and have been changed on disk", modId);
addOutdatedTarget(modId);
Config.INSTANCE.get().extract().put(modId, Config.ExtractionState.OUTDATED);
}
} else {
Path zipPath = Services.PLATFORM.getGlobalFolder().resolve(modId + ".zip");
boolean zipExists = Files.exists(zipPath);
String checksum;
try (FileSystem zipFs = FileSystems.newFileSystem(
URI.create("jar:" + zipPath.toAbsolutePath().toUri()),
Collections.singletonMap("create", "true"))) {
Path outPath = zipFs.getPath("/");
checksum = shouldCopy(defaultResources, outPath, zipExists);
if (checksum != null && !zipExists) {
copyResources(defaultResources, outPath, checksum);
} else if (checksum == null) {
DefaultResources.LOGGER.error("Could not extract default resources for mod {} because they are already extracted and have been changed on disk", modId);
addOutdatedTarget(modId);
Config.INSTANCE.get().extract().put(modId, Config.ExtractionState.OUTDATED);
}
}
if (checksum != null && zipExists) {
Files.delete(zipPath);
try (FileSystem zipFs = FileSystems.newFileSystem(
URI.create("jar:" + Services.PLATFORM.getGlobalFolder().resolve(modId + ".zip").toAbsolutePath().toUri()),
URI.create("jar:" + zipPath.toAbsolutePath().toUri()),
Collections.singletonMap("create", "true"))) {
Path outPath = zipFs.getPath("/");
copyResources(defaultResources, outPath);
copyResources(defaultResources, outPath, checksum);
}
}
}
}
} catch (IOException | RuntimeException e) {
DefaultResources.LOGGER.error("Could not read meta file for mod {}", modId, e);
}
} catch (IOException | RuntimeException e) {
DefaultResources.LOGGER.error("Could not handle default resources for mod {}", modId, e);
}
}
private static synchronized void addOutdatedTarget(String modId) {
OUTDATED_TARGETS.add(modId);
}

private static void copyResources(Path defaultResources, Path outPath) {
public static synchronized boolean isTargetOutdated(String modId) {
return OUTDATED_TARGETS.contains(modId);
}

private static @Nullable String shouldCopy(Path defaultResources, Path outPath, boolean alreadyExists) {
try {
if (alreadyExists) {
Path checksumPath = outPath.resolve(CHECK_FILE_PATH);
String oldChecksum;
if (Files.exists(checksumPath)) {
oldChecksum = Files.readString(checksumPath);
} else {
return null;
}
String newExtractedChecksum = checkPath(outPath);
if (newExtractedChecksum.equals(oldChecksum)) {
String newChecksum = checkPath(defaultResources);
if (newChecksum.equals(oldChecksum)) {
return null;
} else {
return newChecksum;
}
}
} else {
return checkPath(defaultResources);
}
} catch (IOException e) {
DefaultResources.LOGGER.error("Error checking compatibility of resources from {} targeted at {}", defaultResources, outPath, e);
}
return null;
}

@NotNull
private static String checkPath(Path defaultResources) throws IOException {
StringBuilder newChecksum = new StringBuilder();
try (var walk = Files.walk(defaultResources)) {
walk.sorted(Comparator.comparing(p -> defaultResources.relativize(p).toString())).forEach(p -> {
try {
if (!Files.isDirectory(p) && !p.endsWith(CHECK_FILE_PATH)) {
Checksum check = new Adler32();
try (var is = Files.newInputStream(p)) {
byte[] buffer = new byte[BUFFER_SIZE];
int length;
while ((length = is.read(buffer)) > 0) {
check.update(buffer, 0, length);
}
}
newChecksum.append(encode((int) check.getValue()));
}
} catch (IOException e) {
DefaultResources.LOGGER.error("Error calculating checksum at {}", p, e);
}
});
}
return newChecksum.toString();
}

private static CharSequence encode(int i) {
StringBuilder sb = new StringBuilder();
for (int j = 0; j < 4; j++) {
sb.append((char) (((i >> (j * 4)) & 0xF) + 97));
}
return sb;
}

private static void copyResources(Path defaultResources, Path outPath, String checksum) {
try (var walk = Files.walk(defaultResources)) {
walk.forEach(p -> {
walk.sorted(Comparator.comparing(p -> p.relativize(defaultResources).toString())).forEach(p -> {
try {
if (!Files.isDirectory(p)) {
String rel = defaultResources.relativize(p).toString();
Expand All @@ -100,11 +223,13 @@ private static void copyResources(Path defaultResources, Path outPath) {
Files.copy(p, newPath);
}
} catch (IOException e) {
DefaultResources.LOGGER.error(e);
DefaultResources.LOGGER.error("Error checking compatibility of resources from {} targeted at {}, for path {}", defaultResources, outPath, p, e);
}
});
Path checksumPath = outPath.resolve(CHECK_FILE_PATH);
Files.writeString(checksumPath, checksum);
} catch (IOException e) {
DefaultResources.LOGGER.error(e);
DefaultResources.LOGGER.error("Error checking compatibility of resources from {} targeted at {}", defaultResources, outPath, e);
}
}

Expand Down Expand Up @@ -193,14 +318,15 @@ private static List<Pair<String, Pack.ResourcesSupplier>> getDetectedPacks(PackT
return packs;
}

private volatile static boolean GLOBAL_SETUP = false;

public synchronized static void initialize() {
if (!GLOBAL_SETUP) {
GLOBAL_SETUP = true;
Services.PLATFORM.extractResources();
DefaultResources.cleanupExtraction();
}
for (String modId : OUTDATED_TARGETS) {
OUTDATED_RESOURCES_LISTENERS.getOrDefault(modId, List.of()).forEach(OutdatedResourcesListener::resourcesOutdated);
}
}

public synchronized static GlobalResourceManager createStaticResourceManager(PackType type) {
Expand Down
7 changes: 4 additions & 3 deletions common/src/main/java/dev/lukebemish/defaultresources/impl/ModMetaFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
import com.mojang.serialization.Codec;
import com.mojang.serialization.codecs.RecordCodecBuilder;

public record ModMetaFile(String resourcesPath, boolean zip) {
public record ModMetaFile(String resourcesPath, boolean zip, boolean extract) {
public static final Codec<ModMetaFile> CODEC = RecordCodecBuilder.create(instance -> instance.group(
Codec.STRING.fieldOf("resources_path").forGetter(ModMetaFile::resourcesPath),
Codec.BOOL.fieldOf("zip").forGetter(ModMetaFile::zip)
Codec.STRING.optionalFieldOf("resources_path", "defaultresources").forGetter(ModMetaFile::resourcesPath),
Codec.BOOL.optionalFieldOf("zip", true).forGetter(ModMetaFile::zip),
Codec.BOOL.optionalFieldOf("extract", true).forGetter(ModMetaFile::extract)
).apply(instance, ModMetaFile::new));
}
4 changes: 4 additions & 0 deletions ...main/java/dev/lukebemish/defaultresources/impl/fabriquilt/DefaultResourcesFabriQuilt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ public static void forAllMods(BiConsumer<String, Path> consumer) {
FabricLoader.getInstance().getAllMods().forEach(mod -> consumer.accept(mod.getMetadata().getId(), mod.getRootPath()));
}

public static void forAllModsParallel(BiConsumer<String, Path> consumer) {
FabricLoader.getInstance().getAllMods().parallelStream().forEach(mod -> consumer.accept(mod.getMetadata().getId(), mod.getRootPath()));
}

public static void addPackResources(PackType type) {
try {
if (!Files.exists(Services.PLATFORM.getGlobalFolder()))
Expand Down
2 changes: 1 addition & 1 deletion fabriquilt/src/main/java/dev/lukebemish/defaultresources/impl/fabriquilt/PlatformImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public void extractResources() {
} catch (IOException e) {
DefaultResources.LOGGER.error(e);
}
DefaultResourcesFabriQuilt.forAllMods((modID, path) -> {
DefaultResourcesFabriQuilt.forAllModsParallel((modID, path) -> {
if (!modID.equals("minecraft")) {
DefaultResources.forMod(path::resolve, modID);
}
Expand Down
1 change: 1 addition & 0 deletions neoforge/src/main/java/dev/lukebemish/defaultresources/impl/neoforge/PlatformImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public void extractResources() {
}
FMLLoader.getLoadingModList().getModFiles().stream().flatMap(f -> f.getMods().stream())
.filter(PlatformImpl::isExtractable)
.parallel()
.forEach(mod ->
DefaultResources.forMod(mod.getOwningFile().getFile()::findResource, mod.getModId()));
}
Expand Down

0 comments on commit 29f60de

Please sign in to comment.