From 49e928e895fcfa19fd321801caa1a4f3fab88341 Mon Sep 17 00:00:00 2001 From: Oleg Cherednik Date: Sat, 28 Dec 2024 20:36:57 +0300 Subject: [PATCH] #20 Extract entries async --- .../decompose/LocalFileHeaderDecompose.java | 11 +- .../zip4jvm/decompose/Utils.java | 20 +-- .../ConsecutiveAccessDataInputHolder.java | 69 +++++++++ .../zip4jvm/engine/unzip/UnzipEngine.java | 20 ++- .../engine/unzip/UnzipExtractAsyncEngine.java | 134 ++++++++++++++++++ .../engine/unzip/UnzipExtractEngine.java | 105 +++++++------- .../zip4jvm/model/settings/UnzipSettings.java | 33 ++++- .../zip4jvm/utils/quitely/Quietly.java | 4 +- ...eption.java => RunnableWithException.java} | 2 +- .../java/ru/olegcherednik/zip4jvm/Foo.java | 59 +++----- .../zip4jvm/ZipInfoDecomposeTest.java | 19 --- .../zip4jvm/encryption/EncryptionAesTest.java | 5 +- .../encryption/EncryptionPkwareTest.java | 5 +- .../store_split/end_central_directory.data | Bin 22 -> 22 bytes 14 files changed, 340 insertions(+), 146 deletions(-) create mode 100644 src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/ConsecutiveAccessDataInputHolder.java create mode 100644 src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java rename src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/{TaskWithException.java => RunnableWithException.java} (95%) diff --git a/src/main/java/ru/olegcherednik/zip4jvm/decompose/LocalFileHeaderDecompose.java b/src/main/java/ru/olegcherednik/zip4jvm/decompose/LocalFileHeaderDecompose.java index 0181d4417..3667b7d6b 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/decompose/LocalFileHeaderDecompose.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/decompose/LocalFileHeaderDecompose.java @@ -128,26 +128,27 @@ private void copyPayload(Path dir, ZipEntry zipEntry, ZipEntryBlock.LocalFileHea Block content = diagLocalFileHeader.getContent(); long size = zipEntry.getCompressedSize(); - long offs = content.getDiskOffs() + content.getSize(); + // TODO here we should use SrcZip methods + long absOffs = content.getDiskOffs() + content.getSize(); EncryptionMethod encryptionMethod = zipEntry.getEncryptionMethod(); if (encryptionMethod.isAes()) { AesEncryptionHeaderBlock block = (AesEncryptionHeaderBlock) encryptionHeaderBlock; - offs += block.getSalt().getSize(); - offs += block.getPasswordChecksum().getSize(); + absOffs += block.getSalt().getSize(); + absOffs += block.getPasswordChecksum().getSize(); size -= block.getSalt().getSize(); size -= block.getPasswordChecksum().getSize(); size -= block.getMac().getSize(); } else if (encryptionMethod == EncryptionMethod.PKWARE) { PkwareEncryptionHeaderBlock block = (PkwareEncryptionHeaderBlock) encryptionHeaderBlock; - offs += block.getSize(); + absOffs += block.getSize(); size -= block.getSize(); } - Utils.copyLarge(blockModel.getZipModel(), dir.resolve("payload" + EXT_DATA), offs, size); + Utils.copyLarge(blockModel.getZipModel(), dir.resolve("payload" + EXT_DATA), absOffs, absOffs, size); } private EncryptionHeaderDecompose encryptionHeader(EncryptionMethod encryptionMethod, diff --git a/src/main/java/ru/olegcherednik/zip4jvm/decompose/Utils.java b/src/main/java/ru/olegcherednik/zip4jvm/decompose/Utils.java index f182e7846..2f5b0ce39 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/decompose/Utils.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/decompose/Utils.java @@ -50,30 +50,22 @@ public static void print(Path file, Consumer consumer) throws FileN } public static void copyLarge(ZipModel zipModel, Path out, Block block) throws IOException { - copyLarge(zipModel, out, block.getDiskOffs(), block.getSize()); + copyLarge(zipModel, out, block.getDiskOffs(), block.getAbsOffs(), block.getSize()); } - public static void copyLarge(ZipModel zipModel, Path out, long offs, long size) throws IOException { - Path file = zipModel.getSrcZip().getDiskByAbsOffs(offs).getPath(); + public static void copyLarge(ZipModel zipModel, Path out, long diskOffs, long absOffs, long size) + throws IOException { + Path file = zipModel.getSrcZip().getDiskByAbsOffs(absOffs).getPath(); try (InputStream fis = Files.newInputStream(file); OutputStream fos = Files.newOutputStream(out)) { - long skipBytes = fis.skip(offs); - assert skipBytes == offs; + long skipBytes = fis.skip(diskOffs); + assert skipBytes == diskOffs; IOUtils.copyLarge(fis, fos, 0, size); } } - public static void copyByteArray(Path out, byte[] buf, Block block) throws IOException { - ValidationUtils.requireLessOrEqual(block.getAbsOffs(), Integer.MAX_VALUE, "block.absoluteOffs"); - ValidationUtils.requireLessOrEqual(block.getSize(), Integer.MAX_VALUE, "block.size"); - - try (OutputStream fos = Files.newOutputStream(out)) { - fos.write(buf, (int) block.getAbsOffs(), (int) block.getSize()); - } - } - public static void copyByteArray(Path out, byte[] buf) throws IOException { Files.write(out, buf); } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/ConsecutiveAccessDataInputHolder.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/ConsecutiveAccessDataInputHolder.java new file mode 100644 index 000000000..875b65ddf --- /dev/null +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/ConsecutiveAccessDataInputHolder.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package ru.olegcherednik.zip4jvm.engine.unzip; + +import ru.olegcherednik.zip4jvm.io.in.DataInput; +import ru.olegcherednik.zip4jvm.io.in.file.consecutive.ConsecutiveAccessDataInput; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.IOUtils; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; + +/** + * This class is designed to use with custom {@link ExecutorService} only. It holds a list of all {@link DataInput} + * were create in a different threads. When method {@link #release()} is invoked, it closes all created + * {@link DataInput}, but it does not clear {@link #THREAD_LOCAL} for all threads. I.e. after invoking + * {@link #release()} and then invoking {@link #get()}, the given {@link DataInput} will not be {@literal null}, but it + * will be closed and not available to reuse. + * + * @author Oleg Cherednik + * @since 28.12.2024 + */ +@RequiredArgsConstructor +public class ConsecutiveAccessDataInputHolder { + + private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); + + private final List dataInputs = new CopyOnWriteArrayList<>(); + + private final Supplier dataInputSupplier; + + public void release() { + // cannot clear all THREAD_LOCAL here + dataInputs.forEach(IOUtils::closeQuietly); + dataInputs.clear(); + } + + public ConsecutiveAccessDataInput get() { + ConsecutiveAccessDataInput in = THREAD_LOCAL.get(); + + if (in == null) { + in = dataInputSupplier.get(); + THREAD_LOCAL.set(in); + dataInputs.add(in); + } + + return in; + } + +} diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java index ebd7ac679..6ce3a78a0 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java @@ -28,8 +28,8 @@ import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; import ru.olegcherednik.zip4jvm.model.settings.UnzipSettings; import ru.olegcherednik.zip4jvm.model.src.SrcZip; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; -import java.io.IOException; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -47,7 +47,17 @@ public final class UnzipEngine implements ZipFile.Reader { public UnzipEngine(SrcZip srcZip, UnzipSettings settings) { PasswordProvider passwordProvider = settings.getPasswordProvider(); zipModel = ZipModelBuilder.read(srcZip, settings.getCharsetCustomizer(), passwordProvider); - unzipExtractEngine = new UnzipExtractEngine(passwordProvider, zipModel); + unzipExtractEngine = createUnzipExtractEngine(settings, zipModel); + } + + private static UnzipExtractEngine createUnzipExtractEngine(UnzipSettings settings, ZipModel zipModel) { + PasswordProvider passwordProvider = settings.getPasswordProvider(); + + if (settings.getAsyncThreads() == UnzipSettings.ASYNC_THREADS_OFF) + return new UnzipExtractEngine(passwordProvider, zipModel); + + int totalThreads = settings.getAsyncThreads(); + return new UnzipExtractAsyncEngine(passwordProvider, zipModel, totalThreads); } // ---------- ZipFile.Reader ---------- @@ -105,9 +115,9 @@ public ZipFile.Entry next() { }; } - public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) throws IOException { - return srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip) - : new SplitRandomAccessDataInput(srcZip); + public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) { + return Quietly.doRuntime(() -> srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip) + : new SplitRandomAccessDataInput(srcZip)); } } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java new file mode 100644 index 000000000..bcf32de49 --- /dev/null +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package ru.olegcherednik.zip4jvm.engine.unzip; + +import ru.olegcherednik.zip4jvm.model.ZipModel; +import ru.olegcherednik.zip4jvm.model.entry.ZipEntry; +import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; +import ru.olegcherednik.zip4jvm.utils.quitely.functions.RunnableWithException; + +import org.apache.commons.collections4.CollectionUtils; + +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinWorkerThread; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Oleg Cherednik + * @since 28.12.2024 + */ +public class UnzipExtractAsyncEngine extends UnzipExtractEngine { + + protected final int totalThreads; + + public UnzipExtractAsyncEngine(PasswordProvider passwordProvider, ZipModel zipModel, int totalThreads) { + super(passwordProvider, zipModel); + this.totalThreads = totalThreads <= 0 ? Runtime.getRuntime().availableProcessors() : totalThreads; + } + + // ---------- UnzipExtractEngine ---------- + + @Override + protected void extractAllEntries(Path dstDir) { + List> tasks = new LinkedList<>(); + Iterator it = zipModel.absOffsAscIterator(); + + ConsecutiveAccessDataInputHolder dataInputHolder = + new ConsecutiveAccessDataInputHolder(this::createConsecutiveDataInput); + ExecutorService executor = createExecutor(); + + try { + while (it.hasNext()) { + ZipEntry zipEntry = it.next(); + Path file = dstDir.resolve(zipEntry.getFileName()); + + CompletableFuture task = createCompletableFuture( + () -> extractEntry(file, zipEntry, dataInputHolder.get()), executor); + + tasks.add(task); + } + + tasks.forEach(CompletableFuture::join); + } finally { + executor.shutdown(); + dataInputHolder.release(); + } + } + + @Override + protected void extractEntryByPrefix(Path dstDir, Set prefixes) { + assert CollectionUtils.isNotEmpty(prefixes); + + List> tasks = new LinkedList<>(); + Iterator it = zipModel.absOffsAscIterator(); + + ConsecutiveAccessDataInputHolder dataInputHolder = + new ConsecutiveAccessDataInputHolder(this::createConsecutiveDataInput); + ExecutorService executor = createExecutor(); + + try { + while (it.hasNext()) { + ZipEntry zipEntry = it.next(); + String fileName = getFileName(zipEntry, prefixes); + + if (fileName != null) { + Path file = dstDir.resolve(fileName); + CompletableFuture task = createCompletableFuture( + () -> extractEntry(file, zipEntry, dataInputHolder.get()), executor); + + tasks.add(task); + } + } + + tasks.forEach(CompletableFuture::join); + } finally { + dataInputHolder.release(); + executor.shutdown(); + } + } + + // ---------- + + protected ExecutorService createExecutor() { + AtomicInteger counter = new AtomicInteger(); + String format = String.format("zip4jvm-extract-%%0%dd", String.valueOf(totalThreads).length()); + + ForkJoinPool.ForkJoinWorkerThreadFactory factory = pool -> { + ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool); + thread.setName(String.format(format, counter.incrementAndGet())); + return thread; + }; + + return new ForkJoinPool(totalThreads, factory, null, false); + } + + protected CompletableFuture createCompletableFuture(RunnableWithException task, Executor executor) { + return CompletableFuture.runAsync(() -> Quietly.doRuntime(task), executor); + } + +} diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java index 30585e9cd..83ca97cb3 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java @@ -31,6 +31,7 @@ import ru.olegcherednik.zip4jvm.model.password.PasswordProvider; import ru.olegcherednik.zip4jvm.model.src.SrcZip; import ru.olegcherednik.zip4jvm.utils.ZipUtils; +import ru.olegcherednik.zip4jvm.utils.quitely.Quietly; import ru.olegcherednik.zip4jvm.utils.time.DosTimestampConverterUtils; import lombok.RequiredArgsConstructor; @@ -46,13 +47,7 @@ import java.nio.file.Paths; import java.nio.file.attribute.FileTime; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -67,12 +62,12 @@ public class UnzipExtractEngine { protected final ZipModel zipModel; public void extract(Path dstDir, Collection fileNames) { - Map map = null; - - if (CollectionUtils.isNotEmpty(fileNames)) - map = getEntriesByPrefix(new HashSet<>(fileNames)); - - extractEntry(dstDir, map); + if (CollectionUtils.isEmpty(fileNames)) + extractAllEntries(dstDir); + else + extractEntryByPrefix(dstDir, fileNames.stream() + .map(ZipUtils::getFileNameNoDirectoryMarker) + .collect(Collectors.toSet())); } public ZipFile.Entry extract(String fileName) { @@ -81,45 +76,31 @@ public ZipFile.Entry extract(String fileName) { return zipEntry.createImmutableEntry(); } - protected Map getEntriesByPrefix(Set fileNames) { - Map map = new HashMap<>(); - - for (String fileName : fileNames) { - String entryName = ZipUtils.getFileNameNoDirectoryMarker(fileName); + protected void extractAllEntries(Path dstDir) { + try (ConsecutiveAccessDataInput in = createConsecutiveDataInput()) { + Iterator it = zipModel.absOffsAscIterator(); - if (zipModel.hasEntry(entryName)) { - ZipEntry zipEntry = zipModel.getZipEntryByFileName(entryName); - map.put(entryName, FilenameUtils.getName(zipEntry.getFileName())); + while (it.hasNext()) { + ZipEntry zipEntry = it.next(); + Path file = dstDir.resolve(zipEntry.getFileName()); + extractEntry(file, zipEntry, in); } - - for (ZipEntry zipEntry : getEntriesByPrefix(entryName + '/')) - map.put(zipEntry.getFileName(), StringUtils.substring(zipEntry.getFileName(), fileName.length() + 1)); + } catch (IOException e) { + throw new Zip4jvmException(e); } - - return map.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(map); } - protected List getEntriesByPrefix(String prefix) { - return zipModel.getZipEntries().stream() - .filter(entry -> entry.getFileName().startsWith(prefix)) - .collect(Collectors.toList()); - } - - // ---------- + protected void extractEntryByPrefix(Path dstDir, Set prefixes) { + assert CollectionUtils.isNotEmpty(prefixes); - protected void extractEntry(Path dstDir, Map map) { - try (ConsecutiveAccessDataInput in = createConsecutiveDataInput(zipModel.getSrcZip())) { + try (ConsecutiveAccessDataInput in = createConsecutiveDataInput()) { Iterator it = zipModel.absOffsAscIterator(); while (it.hasNext()) { ZipEntry zipEntry = it.next(); + String fileName = getFileName(zipEntry, prefixes); - if (map == null || map.containsKey(zipEntry.getFileName())) { - in.seekForward(zipEntry.getLocalFileHeaderAbsOffs()); - - String fileName = Optional.ofNullable(map) - .map(m -> m.get(zipEntry.getFileName())) - .orElse(zipEntry.getFileName()); + if (fileName != null) { Path file = dstDir.resolve(fileName); extractEntry(file, zipEntry, in); } @@ -129,7 +110,24 @@ protected void extractEntry(Path dstDir, Map map) { } } - protected void extractEntry(Path file, ZipEntry zipEntry, DataInput in) throws IOException { + protected String getFileName(ZipEntry zipEntry, Set prefixes) { + assert CollectionUtils.isNotEmpty(prefixes); + + String fileName = zipEntry.getFileName(); + + if (prefixes.contains(fileName)) + return FilenameUtils.getName(fileName); + + for (String prefix : prefixes) + if (fileName.startsWith(prefix + '/')) + return StringUtils.substring(fileName, prefix.length() + 1); + + return null; + } + + protected void extractEntry(Path file, ZipEntry zipEntry, ConsecutiveAccessDataInput in) throws IOException { + in.seekForward(zipEntry.getLocalFileHeaderAbsOffs()); + if (zipEntry.isSymlink()) extractSymlink(file, zipEntry, in); else if (zipEntry.isDirectory()) @@ -142,7 +140,7 @@ else if (zipEntry.isDirectory()) setFileLastModifiedTime(file, zipEntry); } - protected static void extractSymlink(Path symlink, ZipEntry zipEntry, DataInput in) throws IOException { + protected void extractSymlink(Path symlink, ZipEntry zipEntry, DataInput in) throws IOException { String target = IOUtils.toString(zipEntry.createInputStream(in), Charsets.UTF_8); if (target.startsWith("/")) @@ -154,7 +152,7 @@ else if (target.contains(":")) ZipSymlinkEngine.createRelativeSymlink(symlink, symlink.getParent().resolve(target)); } - protected static void extractEmptyDirectory(Path dir) throws IOException { + protected void extractEmptyDirectory(Path dir) throws IOException { Files.createDirectories(dir); } @@ -164,17 +162,24 @@ protected void extractRegularFile(Path file, ZipEntry zipEntry, DataInput in) th ZipUtils.copyLarge(zipEntry.createInputStream(in), getOutputStream(file)); } - protected static void setFileAttributes(Path path, ZipEntry zipEntry) throws IOException { + public ConsecutiveAccessDataInput createConsecutiveDataInput() { + SrcZip srcZip = zipModel.getSrcZip(); + + return Quietly.doRuntime(() -> srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip) + : new SplitConsecutiveAccessDataInput(srcZip)); + } + + protected void setFileAttributes(Path path, ZipEntry zipEntry) throws IOException { if (zipEntry.getExternalFileAttributes() != null) zipEntry.getExternalFileAttributes().apply(path); } - private static void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException { + protected void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException { long lastModifiedTime = DosTimestampConverterUtils.dosToJavaTime(zipEntry.getLastModifiedTime()); Files.setLastModifiedTime(path, FileTime.fromMillis(lastModifiedTime)); } - protected static OutputStream getOutputStream(Path file) throws IOException { + protected OutputStream getOutputStream(Path file) throws IOException { Path parent = file.getParent(); if (!Files.exists(parent)) @@ -184,12 +189,4 @@ protected static OutputStream getOutputStream(Path file) throws IOException { return Files.newOutputStream(file); } - // ---------- static ---------- - - public static ConsecutiveAccessDataInput createConsecutiveDataInput(SrcZip srcZip) throws IOException { - return srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip) - : new SplitConsecutiveAccessDataInput(srcZip); - - } - } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/model/settings/UnzipSettings.java b/src/main/java/ru/olegcherednik/zip4jvm/model/settings/UnzipSettings.java index e77cf2c30..183c34a9f 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/model/settings/UnzipSettings.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/model/settings/UnzipSettings.java @@ -42,20 +42,28 @@ public final class UnzipSettings { public static final UnzipSettings DEFAULT = builder().build(); + public static final int ASYNC_THREADS_OFF = 0; + public static final int ASYNC_THREADS_AUTO = -1; + private final PasswordProvider passwordProvider; private final Function charsetCustomizer; + private final int asyncThreads; public static Builder builder() { return new Builder(); } public Builder toBuilder() { - return builder().passwordProvider(passwordProvider).charsetCustomizer(charsetCustomizer); + return builder() + .passwordProvider(passwordProvider) + .charsetCustomizer(charsetCustomizer) + .asyncThreads(asyncThreads); } private UnzipSettings(Builder builder) { passwordProvider = builder.passwordProvider; charsetCustomizer = builder.charsetCustomizer; + asyncThreads = builder.asyncThreads; } @NoArgsConstructor(access = AccessLevel.PRIVATE) @@ -65,6 +73,8 @@ public static final class Builder { private PasswordProvider passwordProvider = NoPasswordProvider.INSTANCE; private Function charsetCustomizer = Charsets.UNMODIFIED; + private int asyncThreads = ASYNC_THREADS_AUTO; + public UnzipSettings build() { return new UnzipSettings(this); } @@ -88,12 +98,31 @@ public Builder charset(Charset charset) { return this; } + public Builder async() { + asyncThreadsAuto(); + return this; + } + + public Builder asyncThreads(int asyncThreads) { + this.asyncThreads = Math.max(1, asyncThreads); + return this; + } + + public Builder asyncThreadsAuto() { + asyncThreads = ASYNC_THREADS_AUTO; + return this; + } + + public Builder asyncOff() { + asyncThreads = ASYNC_THREADS_OFF; + return this; + } + private Builder charsetCustomizer(Function charsetCustomizer) { this.charsetCustomizer = Optional.ofNullable(charsetCustomizer).orElse(Charsets.UNMODIFIED); return this; } - } } diff --git a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java index 8c8377fe7..3a9c72a7c 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/Quietly.java @@ -21,8 +21,8 @@ import ru.olegcherednik.zip4jvm.exception.Zip4jvmException; import ru.olegcherednik.zip4jvm.utils.quitely.functions.ByteSupplierWithException; import ru.olegcherednik.zip4jvm.utils.quitely.functions.IntSupplierWithException; +import ru.olegcherednik.zip4jvm.utils.quitely.functions.RunnableWithException; import ru.olegcherednik.zip4jvm.utils.quitely.functions.SupplierWithException; -import ru.olegcherednik.zip4jvm.utils.quitely.functions.TaskWithException; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -64,7 +64,7 @@ public static byte doRuntime(ByteSupplierWithException supplier) { } } - public static void doRuntime(TaskWithException task) { + public static void doRuntime(RunnableWithException task) { try { task.run(); } catch (Zip4jvmException e) { diff --git a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java similarity index 95% rename from src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java rename to src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java index e01728d57..644a70b20 100644 --- a/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/TaskWithException.java +++ b/src/main/java/ru/olegcherednik/zip4jvm/utils/quitely/functions/RunnableWithException.java @@ -23,7 +23,7 @@ * @since 26.02.2023 */ @FunctionalInterface -public interface TaskWithException { +public interface RunnableWithException { void run() throws Exception; diff --git a/src/test/java/ru/olegcherednik/zip4jvm/Foo.java b/src/test/java/ru/olegcherednik/zip4jvm/Foo.java index 402a8a1a6..9cc750d55 100644 --- a/src/test/java/ru/olegcherednik/zip4jvm/Foo.java +++ b/src/test/java/ru/olegcherednik/zip4jvm/Foo.java @@ -18,12 +18,15 @@ */ package ru.olegcherednik.zip4jvm; -import ru.olegcherednik.zip4jvm.model.settings.ZipInfoSettings; +import ru.olegcherednik.zip4jvm.model.settings.UnzipSettings; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; +import java.util.TreeSet; /** * @author Oleg Cherednik @@ -34,51 +37,23 @@ public class Foo { public static void main(String[] args) throws IOException { final long timeFrom = System.currentTimeMillis(); - int[][] token = new int[3][3]; - -// Path zip = Paths.get("d:/zip4jvm/zip64/split/ferdinand.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/split/ducati.zip"); - -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-ecd.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/bikes.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-dcl.zip"); -// Path zip = Paths.get("d:/zip4jvm/aaa/app.apk"); -// Path zip = Paths.get("d:/zip4jvm/aaa/android.apk"); -// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip"); -// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip"); + Path dstDir = Paths.get("f:/zip4jvm/zip64/multi/out"); + Path zip = Paths.get("f:/zip4jvm/zip64/multi/aes_10k.zip"); -// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip"); -// Path zip = Paths.get("d:/Programming/GitHub/zip4jvm/src/test/resources/secure-zip/strong/store_solid_aes256_strong_ecd.zip"); + FileUtils.deleteDirectory(dstDir.toFile()); - //Path zip = Paths.get("d:/zip4jvm/zip64/src.zip"); -// Path zip = Paths.get("d:/zip4jvm/scd/aes256bit.zip"); -// Path zip = Paths.get("d:/zip4jvm/scd/P1AA4B3C.zip"); - Path zip = Paths.get("d:/zip4jvm/scd/onetwo.zip"); -// Path zip = Paths.get("D:/Programming/GitHub/zip4jvm/src/test/resources/symlink/win/unique-symlink-target.zip"); - Path dstDir = Paths.get("d:/zip4jvm/scd/xxx"); + UnzipSettings settings = UnzipSettings.builder() + .asyncThreadsAuto() + .build(); -// ZipIt.zip(zip).settings(settings).add(dirSrcData); - - -// for (Path zip : Arrays.asList(zip1, zip2)) { -// System.out.println(zip); -// UnzipIt.zip(zip).dstDir(dstDir) -// .settings(UnzipSettings.builder() -// .password(password) -// .build()) -// .extract(); -// ZipInfo.zip(zip).password("1".toCharArray()).printShortInfo(); - ZipInfo.zip(zip) - .settings(ZipInfoSettings.builder() - .copyPayload(true) - .readEntries(true) - .build()) - .password("1".toCharArray()) - .decompose(Paths.get(dstDir.toString(), zip.getFileName().toString())); + UnzipIt.zip(zip).settings(settings).dstDir(dstDir).extract(); final long timeTo = System.currentTimeMillis(); - System.out.format("Time: %d sec", TimeUnit.MILLISECONDS.toSeconds(timeTo - timeFrom)); + long millis = timeTo - timeFrom; + long minutes = (millis / 1000) / 60; + int seconds = (int) ((millis / 1000) % 60); + System.out.format("Time: %02d:%02d", minutes, seconds); } + } diff --git a/src/test/java/ru/olegcherednik/zip4jvm/ZipInfoDecomposeTest.java b/src/test/java/ru/olegcherednik/zip4jvm/ZipInfoDecomposeTest.java index 1724a9bb7..a92011b35 100644 --- a/src/test/java/ru/olegcherednik/zip4jvm/ZipInfoDecomposeTest.java +++ b/src/test/java/ru/olegcherednik/zip4jvm/ZipInfoDecomposeTest.java @@ -50,84 +50,72 @@ public static void removeDir() throws IOException { public void shouldDecomposeWhenStoreSolid() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSolid).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_solid"); } public void shouldDecomposeWhenStoreSolidPkware() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSolidPkware).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_solid_pkware"); } public void shouldDecomposeWhenStoreSolidAes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSolidAes).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_solid_aes"); } public void shouldDecomposeWhenStoreSplit() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSplit).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_split"); } public void shouldDecomposeWhenStoreSplitPkware() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSplitPkware).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_split_pkware"); } public void shouldDecomposeWhenStoreSplitAes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.zipStoreSplitAes).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/store_split_aes"); } public void shouldDecomposeWhenSingleItemZip() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(Zip4jvmSuite.getResourcePath("zip/single_item.zip")).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/single_item"); } public void shouldDecomposeWhenStrongStoreAes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipStoreSolidAes256StrongZip).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_store_aes"); } public void shouldDecomposeWhenStrongDeflateAes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipDeflateSolidAes256StrongZip).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_deflate_aes"); } public void shouldDecomposeWhenStrongBzip2Aes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipBzip2SolidAes256StrongZip).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_bzip2_aes"); } public void shouldDecomposeWhenStrongDeflate64Aes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipDeflate64SolidAes256StrongZip).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_deflate64_aes"); } public void shouldDecomposeWhenStrongLzmaAes() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipLzmaSolidAes256StrongZip).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_lzma_aes"); } @@ -136,7 +124,6 @@ public void shouldDecomposeWhenStrongLzmaAes() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongStoreAesEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipStoreSolidAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_store_aes_ecd"); } @@ -144,7 +131,6 @@ public void shouldDecomposeWhenStrongStoreAesEcd() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongDeflateAesEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipDeflateSolidAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_deflate_aes_ecd"); } @@ -152,7 +138,6 @@ public void shouldDecomposeWhenStrongDeflateAesEcd() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongBzip2AesEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipBzip2SolidAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_bzip2_aes_ecd"); } @@ -160,7 +145,6 @@ public void shouldDecomposeWhenStrongBzip2AesEcd() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongDeflate64AesEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipDeflate64SolidAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_deflate64_aes_ecd"); } @@ -168,14 +152,12 @@ public void shouldDecomposeWhenStrongDeflate64AesEcd() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongLzmaAesEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipLzmaSolidAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_lzma_aes_ecd"); } public void shouldDecomposeWhenStrongBzip2AesSplit() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipBzip2SplitAes256StrongZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/strong_bzip2_aes_split"); } @@ -183,7 +165,6 @@ public void shouldDecomposeWhenStrongBzip2AesSplit() throws IOException { @Test(enabled = false) public void shouldDecomposeWhenStrongBzip2AesSplitEcd() throws IOException { Path dir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); - Files.createDirectories(dir.getParent()); ZipInfo.zip(TestData.secureZipBzip2SplitAes256StrongEcdZip).password(password).decompose(dir); assertThatDirectory(dir).matchesResourceDirectory("/decompose/strong/ecd/strong_bzip2_aes_split_ecd"); } diff --git a/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionAesTest.java b/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionAesTest.java index 8fdab894f..542886c26 100644 --- a/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionAesTest.java +++ b/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionAesTest.java @@ -158,7 +158,10 @@ public void shouldThrowExceptionWhenUnzipAesEncryptedZipWithIncorrectPassword() Path dstDir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); char[] password = UUID.randomUUID().toString().toCharArray(); - UnzipSettings settings = UnzipSettings.builder().password(password).build(); + UnzipSettings settings = UnzipSettings.builder() + .password(password) + .asyncOff() + .build(); assertThatThrownBy(() -> UnzipIt.zip(zipStoreSplitAes).dstDir(dstDir).settings(settings).extract()) .isExactlyInstanceOf(IncorrectZipEntryPasswordException.class); diff --git a/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionPkwareTest.java b/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionPkwareTest.java index ef441bd84..1d42f6c97 100644 --- a/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionPkwareTest.java +++ b/src/test/java/ru/olegcherednik/zip4jvm/encryption/EncryptionPkwareTest.java @@ -126,7 +126,10 @@ public void shouldThrowExceptionWhenUnzipPkwareEncryptedZipWithIncorrectPassword Path dstDir = Zip4jvmSuite.subDirNameAsMethodName(ROOT_DIR); char[] password = UUID.randomUUID().toString().toCharArray(); - UnzipSettings settings = UnzipSettings.builder().password(password).build(); + UnzipSettings settings = UnzipSettings.builder() + .password(password) + .asyncOff() + .build(); assertThatThrownBy(() -> UnzipIt.zip(zipStoreSplitPkware).dstDir(dstDir).settings(settings).extract()) .isExactlyInstanceOf(IncorrectZipEntryPasswordException.class); diff --git a/src/test/resources/decompose/store_split/end_central_directory.data b/src/test/resources/decompose/store_split/end_central_directory.data index 684e0418c767801156abbd8fd68c3dbb6b15a796..d304473a2e779169250c109a811d68ac58b5bac0 100644 GIT binary patch literal 22 acmWIWW@Te#U}fM1!Ukpr2C)S|f&l;+x&j9P literal 22 ecmcay6M8P