diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/DataInputThreadLocal.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/DataInputThreadLocal.java new file mode 100644 index 000000000..993676af8 --- /dev/null +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/DataInputThreadLocal.java @@ -0,0 +1,42 @@ +package ru.olegcherednik.zip4jvm.engine.unzip; + +import ru.olegcherednik.zip4jvm.io.in.DataInput; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.IOUtils; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Supplier; + +/** + * @author Oleg Cherednik + * @since 28.12.2024 + */ +@RequiredArgsConstructor +public class DataInputThreadLocal extends ThreadLocal { + + private final Supplier dataInputSup; + private final List dataInputs = new CopyOnWriteArrayList<>(); + + public void release() { + dataInputs.forEach(IOUtils::closeQuietly); + dataInputs.clear(); + } + + // ---------- ThreadLocal ---------- + + @Override + public T get() { + T in = super.get(); + + if (in == null) { + in = dataInputSup.get(); + set(in); + dataInputs.add(in); + } + + return in; + } + +} diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java index ebd7ac679..d445de569 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java @@ -28,8 +28,8 @@ import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; import ru.olegcherednik.zip4jvm.model.settings.UnzipSettings; import ru.olegcherednik.zip4jvm.model.src.SrcZip; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; -import java.io.IOException; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -47,7 +47,7 @@ public final class UnzipEngine implements ZipFile.Reader { public UnzipEngine(SrcZip srcZip, UnzipSettings settings) { PasswordProvider passwordProvider = settings.getPasswordProvider(); zipModel = ZipModelBuilder.read(srcZip, settings.getCharsetCustomizer(), passwordProvider); - unzipExtractEngine = new UnzipExtractEngine(passwordProvider, zipModel); + unzipExtractEngine = new UnzipExtractAsyncEngine(passwordProvider, zipModel); } // ---------- ZipFile.Reader ---------- @@ -105,9 +105,11 @@ public ZipFile.Entry next() { }; } - public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) throws IOException { - return srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip) - : new SplitRandomAccessDataInput(srcZip); + // ---------- static ---------- + + public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) { + return Quietly.doRuntime(() -> srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip) + : new SplitRandomAccessDataInput(srcZip)); } } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java new file mode 100644 index 000000000..43939b0e6 --- /dev/null +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package ru.olegcherednik.zip4jvm.engine.unzip; + +import ru.olegcherednik.zip4jvm.io.in.file.consecutive.ConsecutiveAccessDataInput; +import ru.olegcherednik.zip4jvm.model.ZipModel; +import ru.olegcherednik.zip4jvm.model.entry.ZipEntry; +import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; +import ru.olegcherednik.zip4jvm.utils.quitely.functions.RunnableWithException; + +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinWorkerThread; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Oleg Cherednik + * @since 28.12.2024 + */ +public class UnzipExtractAsyncEngine extends UnzipExtractEngine { + + public UnzipExtractAsyncEngine(PasswordProvider passwordProvider, ZipModel zipModel) { + super(passwordProvider, zipModel); + } + + // ---------- UnzipExtractEngine ---------- + + @Override + protected void extractEntry(Path dstDir, Map map) { + List> tasks = new LinkedList<>(); + Iterator it = zipModel.absOffsAscIterator(); + + DataInputThreadLocal threadLocalDataInput = + new DataInputThreadLocal<>(this::createConsecutiveDataInput); + Executor executor = createExecutor(); + + try { + + while (it.hasNext()) { + ZipEntry zipEntry = it.next(); + + if (map != null && !map.containsKey(zipEntry.getFileName())) + continue; + + String fileName = Optional.ofNullable(map) + .map(m -> m.get(zipEntry.getFileName())) + .orElse(zipEntry.getFileName()); + Path file = dstDir.resolve(fileName); + + CompletableFuture task = createCompletableFuture( + () -> extractEntry(file, zipEntry, threadLocalDataInput.get()), executor); + + tasks.add(task); + } + + tasks.forEach(CompletableFuture::join); + } finally { + threadLocalDataInput.release(); + } + } + + protected Executor createExecutor() { + AtomicInteger counter = new AtomicInteger(); + + ForkJoinPool.ForkJoinWorkerThreadFactory factory = pool -> { + ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool); + thread.setName(String.format("zip4jvm-extract-%02d", counter.incrementAndGet())); + return thread; + }; + + return new ForkJoinPool(Runtime.getRuntime().availableProcessors(), factory, null, false); + } + + protected CompletableFuture createCompletableFuture(RunnableWithException task, Executor executor) { + return CompletableFuture.runAsync(() -> Quietly.doRuntime(task), executor); + } + +} diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java index 30585e9cd..17b8b1342 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java @@ -31,6 +31,7 @@ import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; import ru.olegcherednik.zip4jvm.model.src.SrcZip; import ru.olegcherednik.zip4jvm.utils.ZipUtils; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; import ru.olegcherednik.zip4jvm.utils.time.DosTimestampConverterUtils; import lombok.RequiredArgsConstructor; @@ -105,31 +106,31 @@ protected List getEntriesByPrefix(String prefix) { .collect(Collectors.toList()); } - // ---------- - protected void extractEntry(Path dstDir, Map map) { - try (ConsecutiveAccessDataInput in = createConsecutiveDataInput(zipModel.getSrcZip())) { + try (ConsecutiveAccessDataInput in = createConsecutiveDataInput()) { Iterator it = zipModel.absOffsAscIterator(); while (it.hasNext()) { ZipEntry zipEntry = it.next(); - if (map == null || map.containsKey(zipEntry.getFileName())) { - in.seekForward(zipEntry.getLocalFileHeaderAbsOffs()); + if (map != null && !map.containsKey(zipEntry.getFileName())) + continue; + + String fileName = Optional.ofNullable(map) + .map(m -> m.get(zipEntry.getFileName())) + .orElse(zipEntry.getFileName()); + Path file = dstDir.resolve(fileName); - String fileName = Optional.ofNullable(map) - .map(m -> m.get(zipEntry.getFileName())) - .orElse(zipEntry.getFileName()); - Path file = dstDir.resolve(fileName); - extractEntry(file, zipEntry, in); - } + extractEntry(file, zipEntry, in); } } catch (IOException e) { throw new Zip4jvmException(e); } } - protected void extractEntry(Path file, ZipEntry zipEntry, DataInput in) throws IOException { + protected void extractEntry(Path file, ZipEntry zipEntry, ConsecutiveAccessDataInput in) throws IOException { + in.seekForward(zipEntry.getLocalFileHeaderAbsOffs()); + if (zipEntry.isSymlink()) extractSymlink(file, zipEntry, in); else if (zipEntry.isDirectory()) @@ -142,7 +143,7 @@ else if (zipEntry.isDirectory()) setFileLastModifiedTime(file, zipEntry); } - protected static void extractSymlink(Path symlink, ZipEntry zipEntry, DataInput in) throws IOException { + protected void extractSymlink(Path symlink, ZipEntry zipEntry, DataInput in) throws IOException { String target = IOUtils.toString(zipEntry.createInputStream(in), Charsets.UTF_8); if (target.startsWith("/")) @@ -154,7 +155,7 @@ else if (target.contains(":")) ZipSymlinkEngine.createRelativeSymlink(symlink, symlink.getParent().resolve(target)); } - protected static void extractEmptyDirectory(Path dir) throws IOException { + protected void extractEmptyDirectory(Path dir) throws IOException { Files.createDirectories(dir); } @@ -164,17 +165,26 @@ protected void extractRegularFile(Path file, ZipEntry zipEntry, DataInput in) th ZipUtils.copyLarge(zipEntry.createInputStream(in), getOutputStream(file)); } - protected static void setFileAttributes(Path path, ZipEntry zipEntry) throws IOException { + public ConsecutiveAccessDataInput createConsecutiveDataInput() { + return Quietly.doRuntime(() -> { + SrcZip srcZip = zipModel.getSrcZip(); + + return srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip) + : new SplitConsecutiveAccessDataInput(srcZip); + }); + } + + protected void setFileAttributes(Path path, ZipEntry zipEntry) throws IOException { if (zipEntry.getExternalFileAttributes() != null) zipEntry.getExternalFileAttributes().apply(path); } - private static void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException { + protected void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException { long lastModifiedTime = DosTimestampConverterUtils.dosToJavaTime(zipEntry.getLastModifiedTime()); Files.setLastModifiedTime(path, FileTime.fromMillis(lastModifiedTime)); } - protected static OutputStream getOutputStream(Path file) throws IOException { + protected OutputStream getOutputStream(Path file) throws IOException { Path parent = file.getParent(); if (!Files.exists(parent)) @@ -184,12 +194,4 @@ protected static OutputStream getOutputStream(Path file) throws IOException { return Files.newOutputStream(file); } - // ---------- static ---------- - - public static ConsecutiveAccessDataInput createConsecutiveDataInput(SrcZip srcZip) throws IOException { - return srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip) - : new SplitConsecutiveAccessDataInput(srcZip); - - } - } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/in/file/consecutive/SolidConsecutiveAccessDataInput.java b/src/main/java/ru/olegcherednik/zip4jvm/io/in/file/consecutive/SolidConsecutiveAccessDataInput.java index a2660ef2e..0ec43fa9b 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/io/in/file/consecutive/SolidConsecutiveAccessDataInput.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/io/in/file/consecutive/SolidConsecutiveAccessDataInput.java @@ -42,6 +42,7 @@ public class SolidConsecutiveAccessDataInput extends BaseConsecutiveAccessDataIn private final InputStream in; public SolidConsecutiveAccessDataInput(SrcZip srcZip) throws IOException { + System.out.println(Thread.currentThread().getName()); byteOrder = srcZip.getByteOrder(); in = new BufferedInputStream(Files.newInputStream(srcZip.getDiskByNo(0).getPath())); } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java index 8c8377fe7..e9bf41477 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java @@ -22,7 +22,7 @@ import ru.olegcherednik.zip4jvm.utils.quitely.functions.ByteSupplierWithException; import ru.olegcherednik.zip4jvm.utils.quitely.functions.IntSupplierWithException; import ru.olegcherednik.zip4jvm.utils.quitely.functions.SupplierWithException; -import ru.olegcherednik.zip4jvm.utils.quitely.functions.TaskWithException; +import ru.olegcherednik.zip4jvm.utils.quitely.functions.RunnableWithException; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -64,7 +64,7 @@ public static byte doRuntime(ByteSupplierWithException supplier) { } } - public static void doRuntime(TaskWithException task) { + public static void doRuntime(RunnableWithException task) { try { task.run(); } catch (Zip4jvmException e) { diff --git a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java similarity index 95% rename from src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java rename to src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java index e01728d57..644a70b20 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java @@ -23,7 +23,7 @@ * @since 26.02.2023 */ @FunctionalInterface -public interface TaskWithException { +public interface RunnableWithException { void run() throws Exception; diff --git a/src/test/java/ru/olegcherednik/zip4jvm/Foo.java b/src/test/java/ru/olegcherednik/zip4jvm/Foo.java index 402a8a1a6..b8f8786c8 100644 --- a/src/test/java/ru/olegcherednik/zip4jvm/Foo.java +++ b/src/test/java/ru/olegcherednik/zip4jvm/Foo.java @@ -18,12 +18,9 @@ */ package ru.olegcherednik.zip4jvm; -import ru.olegcherednik.zip4jvm.model.settings.ZipInfoSettings; - import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; /** * @author Oleg Cherednik @@ -34,51 +31,17 @@ public class Foo { public static void main(String[] args) throws IOException { final long timeFrom = System.currentTimeMillis(); - int[][] token = new int[3][3]; - -// Path zip = Paths.get("d:/zip4jvm/zip64/split/ferdinand.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/split/ducati.zip"); - -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-ecd.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/bikes.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-dcl.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/app.apk"); -// Path zip = Paths.get("d:/zip4jvm/aaa/android.apk"); - -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip"); -// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip"); -// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip"); -// Path zip = Paths.get("d:/Programming/GitHub/zip4jvm/src/test/resources/secure-zip/strong/store_solid_aes256_strong_ecd.zip"); + Path zip = Paths.get("d:/zip4jvm/zip64/multi/aes_10k.zip"); + Path dstDir = Paths.get("d:/zip4jvm/zip64/multi/out"); - //Path zip = Paths.get("d:/zip4jvm/zip64/src.zip"); -// Path zip = Paths.get("d:/zip4jvm/scd/aes256bit.zip"); -// Path zip = Paths.get("d:/zip4jvm/scd/P1AA4B3C.zip"); - Path zip = Paths.get("d:/zip4jvm/scd/onetwo.zip"); -// Path zip = Paths.get("D:/Programming/GitHub/zip4jvm/src/test/resources/symlink/win/unique-symlink-target.zip"); - Path dstDir = Paths.get("d:/zip4jvm/scd/xxx"); - -// ZipIt.zip(zip).settings(settings).add(dirSrcData); - - -// for (Path zip : Arrays.asList(zip1, zip2)) { -// System.out.println(zip); -// UnzipIt.zip(zip).dstDir(dstDir) -// .settings(UnzipSettings.builder() -// .password(password) -// .build()) -// .extract(); -// ZipInfo.zip(zip).password("1".toCharArray()).printShortInfo(); - ZipInfo.zip(zip) - .settings(ZipInfoSettings.builder() - .copyPayload(true) - .readEntries(true) - .build()) - .password("1".toCharArray()) - .decompose(Paths.get(dstDir.toString(), zip.getFileName().toString())); + UnzipIt.zip(zip).dstDir(dstDir).extract(); final long timeTo = System.currentTimeMillis(); - System.out.format("Time: %d sec", TimeUnit.MILLISECONDS.toSeconds(timeTo - timeFrom)); + long millis = timeTo - timeFrom; + long minutes = (millis / 1000) / 60; + int seconds = (int) ((millis / 1000) % 60); + System.out.format("Time: %02d:%02d", minutes, seconds); } + }