#3 store compression size optimization

oleg-cherednik · Nov 1, 2024 · 6c80d4b · 6c80d4b
1 parent fa03abc
commit 6c80d4b
Show file tree

Hide file tree

Showing 32 changed files with 517 additions and 71 deletions.
diff --git a/gradle/pmd.gradle b/gradle/pmd.gradle
@@ -1,5 +1,10 @@
 apply plugin: 'pmd'
 
+dependencies {
+    pmd 'net.sourceforge.pmd:pmd-ant:7.0.0-rc1'
+    pmd 'net.sourceforge.pmd:pmd-java:7.0.0-rc1'
+}
+
 pmd {
     toolVersion = '7.0.0'
     consoleOutput = false

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareEncoder.java b/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareEncoder.java
@@ -25,6 +25,7 @@
 import lombok.RequiredArgsConstructor;
 
 import java.io.IOException;
+import java.util.function.IntSupplier;
 
 import static ru.olegcherednik.zip4jvm.utils.ValidationUtils.requireNotEmpty;
 
@@ -36,21 +37,22 @@
 public final class PkwareEncoder implements Encoder {
 
     private final PkwareEngine engine;
-    private final PkwareHeader header;
+    private final IntSupplier key;
 
     public static PkwareEncoder create(ZipEntry entry) {
         requireNotEmpty(entry.getPassword(), entry.getFileName() + ".password");
 
         PkwareEngine engine = new PkwareEngine(entry.getPassword());
-        PkwareHeader header = PkwareHeader.create(engine, entry.getLastModifiedTime());
-        return new PkwareEncoder(engine, header);
+
+        return new PkwareEncoder(engine, () -> entry.isDataDescriptorAvailable() ? entry.getLastModifiedTime()
+                                                                                 : (int) entry.getChecksum() >> 16);
     }
 
     // ---------- Encoder ----------
 
     @Override
     public void writeEncryptionHeader(DataOutput out) throws IOException {
-        header.write(out);
+        PkwareHeader.create(engine, key.getAsInt()).write(out);
     }
 
     // ---------- Encrypt ----------

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareHeader.java b/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareHeader.java
@@ -40,16 +40,16 @@ public final class PkwareHeader {
 
     private final byte[] buf;
 
-    public static PkwareHeader create(PkwareEngine engine, int checksum) {
-        return new PkwareHeader(createBuf(engine, checksum));
+    public static PkwareHeader create(PkwareEngine engine, int key) {
+        return new PkwareHeader(createBuf(engine, key & 0xFFFF));
     }
 
-    private static byte[] createBuf(PkwareEngine engine, int checksum) {
+    private static byte[] createBuf(PkwareEngine engine, int key) {
         byte[] buf = new byte[SIZE];
 
         new SecureRandom().nextBytes(buf);
-        buf[buf.length - 1] = low(checksum);
-        buf[buf.length - 2] = high(checksum);
+        buf[buf.length - 1] = low(key);
+        buf[buf.length - 2] = high(key);
         engine.encrypt(buf, 0, buf.length);
 
         return buf;

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/ZipEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/ZipEngine.java
@@ -26,6 +26,7 @@
 import ru.olegcherednik.zip4jvm.io.out.data.SolidZipDataOutput;
 import ru.olegcherednik.zip4jvm.io.out.data.SplitZipDataOutput;
 import ru.olegcherednik.zip4jvm.io.writers.ExistedEntryWriter;
+import ru.olegcherednik.zip4jvm.io.writers.ZipEntryNoDataDescriptorWriter;
 import ru.olegcherednik.zip4jvm.io.writers.ZipEntryWriter;
 import ru.olegcherednik.zip4jvm.model.ZipModel;
 import ru.olegcherednik.zip4jvm.model.builders.ZipModelBuilder;
@@ -37,6 +38,7 @@
 import ru.olegcherednik.zip4jvm.utils.PathUtils;
 import ru.olegcherednik.zip4jvm.utils.ZipUtils;
 import ru.olegcherednik.zip4jvm.utils.function.Writer;
+import ru.olegcherednik.zip4jvm.utils.quitely.Quietly;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -49,6 +51,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static ru.olegcherednik.zip4jvm.utils.ValidationUtils.requireNotBlank;
@@ -67,11 +70,11 @@ public final class ZipEngine implements ZipFile.Writer {
     private final ZipSettings settings;
     private final Map<String, Writer> fileNameWriter = new LinkedHashMap<>();
 
-    public ZipEngine(Path zip, ZipSettings settings) throws IOException {
-        this.zip = zip;
+    public ZipEngine(Path zip, ZipSettings settings) {
+        this.zip = requireNotNull(zip, "ZipEngine.zip");
+        this.settings = requireNotNull(settings, "ZipEngine.settings");
         tempZipModel = createTempZipModel(zip, settings, fileNameWriter);
         zipSymlinkEngine = new ZipSymlinkEngine(settings.getZipSymlink());
-        this.settings = settings;
     }
 
     @Override
@@ -137,12 +140,20 @@ public void add(ZipFile.Entry entry) {
         add(zipEntry);
     }
 
-    private void add(ZipEntry zipEntry) {
-        if (fileNameWriter.containsKey(zipEntry.getFileName()))
-            throw new EntryDuplicationException(zipEntry.getFileName());
+    private void add(ZipEntry entry) {
+        if (fileNameWriter.containsKey(entry.getFileName()))
+            throw new EntryDuplicationException(entry.getFileName());
 
-        tempZipModel.addEntry(zipEntry);
-        fileNameWriter.put(zipEntry.getFileName(), new ZipEntryWriter(zipEntry));
+        tempZipModel.addEntry(entry);
+        fileNameWriter.put(entry.getFileName(), createWriter(entry));
+    }
+
+    private Writer createWriter(ZipEntry entry) {
+        if (entry.isDataDescriptorAvailable())
+            return new ZipEntryWriter(entry);
+
+        Path dir = tempZipModel.getTempDir().resolve(UUID.randomUUID().toString());
+        return new ZipEntryNoDataDescriptorWriter(entry, dir);
     }
 
     @Override
@@ -230,10 +241,10 @@ private void moveTempZipFiles() throws IOException {
         Files.deleteIfExists(tempZipModel.getSrcZip().getPath().getParent());
     }
 
-    private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<String, Writer> fileNameWriter)
-            throws IOException {
+    private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<String, Writer> fileNameWriter) {
         Path tempZip = createTempZip(zip);
         ZipModel tempZipModel = ZipModelBuilder.build(tempZip, settings);
+        tempZipModel.setTempDir(tempZip.getParent());
 
         if (Files.exists(zip)) {
             ZipModel zipModel = ZipModelBuilder.read(SrcZip.of(zip));
@@ -254,10 +265,12 @@ private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<S
         return tempZipModel;
     }
 
-    private static Path createTempZip(Path zip) throws IOException {
-        Path dir = zip.getParent().resolve("tmp");
-        Files.createDirectories(dir);
-        return dir.resolve(zip.getFileName());
+    private static Path createTempZip(Path zip) {
+        return Quietly.doQuietly(() -> {
+            Path dir = zip.getParent().resolve("tmp");
+            Files.createDirectories(dir);
+            return dir.resolve(zip.getFileName());
+        });
     }
 
     private static DataOutput creatDataOutput(ZipModel zipModel) throws IOException {

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/DataOutputStream.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/DataOutputStream.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ru.olegcherednik.zip4jvm.io.out;
+
+import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
+
+import lombok.RequiredArgsConstructor;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author Oleg Cherednik
+ * @since 29.10.2024
+ */
+@RequiredArgsConstructor
+public class DataOutputStream extends OutputStream {
+
+    private final DataOutput out;
+
+    @Override
+    public void write(int val) throws IOException {
+        out.writeByte(val);
+    }
+
+    @Override
+    public void write(byte[] buf, int offs, int len) throws IOException {
+        out.write(buf, offs, len);
+    }
+
+    @Override
+    public void flush() throws IOException {
+        out.flush();
+    }
+
+    @Override
+    public void close() throws IOException {
+        out.close();
+    }
+
+}
diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/EncryptedDataOutput.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/EncryptedDataOutput.java
@@ -48,6 +48,10 @@ public void writeEncryptionHeader() throws IOException {
         encoder.writeEncryptionHeader(out);
     }
 
+    public void writeEncryptionHeader(DataOutput out) throws IOException {
+        encoder.writeEncryptionHeader(out);
+    }
+
     public void encodingAccomplished() throws IOException {
         encoder.close(out);
     }

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/WriteFileDataOutput.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/WriteFileDataOutput.java
@@ -35,11 +35,11 @@
  * @author Oleg Cherednik
  * @since 11.02.2020
  */
-abstract class WriteFileDataOutput extends BaseDataOutput {
+public class WriteFileDataOutput extends BaseDataOutput {
 
     private WriteFile writeFile;
 
-    protected final void createFile(Path zip) throws IOException {
+    public final void createFile(Path zip) throws IOException {
         writeFile = LittleEndianWriteFile.create(zip);
     }
 

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/PayloadCalculationOutputStream.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/PayloadCalculationOutputStream.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package ru.olegcherednik.zip4jvm.io.out.entry;
 
 import ru.olegcherednik.zip4jvm.model.entry.ZipEntry;

diff --git a/...try/encrypted/Bzip2EntryOutputStream.java → ...ry/compressed/Bzip2EntryOutputStream.java b/...try/encrypted/Bzip2EntryOutputStream.java → ...ry/compressed/Bzip2EntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.io.bzip2.Bzip2OutputStream;
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;

diff --git a/...ncrypted/CompressedEntryOutputStream.java → ...mpressed/CompressedEntryOutputStream.java b/...ncrypted/CompressedEntryOutputStream.java → ...mpressed/CompressedEntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.exception.CompressionNotSupportedException;
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
@@ -36,6 +36,9 @@
  * This class represents a compressed stream using given {@link CompressionMethod}.
  * It extends from the {@link OutputStream} to be able to use standard output
  * optimizations (e.g. from {@link IOUtils}).
+ * <p>
+ * This {@link OutputStream} does not close delegate {@link DataOutput} when
+ * method {@link CompressedEntryOutputStream#close()} is invoked.
  *
  * @author Oleg Cherednik
  * @since 12.02.2020
@@ -71,7 +74,7 @@ public final void write(int b) throws IOException {
 
     @Override
     public void write(byte[] buf, int offs, int len) throws IOException {
-        throw new NotImplementedException("CompressedEntryOutputStream.write(byte[], int, int)");
+        throw new NotImplementedException(getClass().getSimpleName() + ".write(byte[], int, int)");
     }
 
 }
diff --git a/...y/encrypted/DeflateEntryOutputStream.java → .../compressed/DeflateEntryOutputStream.java b/...y/encrypted/DeflateEntryOutputStream.java → .../compressed/DeflateEntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
 import ru.olegcherednik.zip4jvm.model.CompressionLevel;

diff --git a/...ntry/encrypted/LzmaEntryOutputStream.java → ...try/compressed/LzmaEntryOutputStream.java b/...ntry/encrypted/LzmaEntryOutputStream.java → ...try/compressed/LzmaEntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.io.lzma.LzmaInputStream;
 import ru.olegcherednik.zip4jvm.io.lzma.LzmaOutputStream;

diff --git a/...try/encrypted/StoreEntryOutputStream.java → ...ry/compressed/StoreEntryOutputStream.java b/...try/encrypted/StoreEntryOutputStream.java → ...ry/compressed/StoreEntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
 

diff --git a/...ntry/encrypted/ZstdEntryOutputStream.java → ...try/compressed/ZstdEntryOutputStream.java b/...ntry/encrypted/ZstdEntryOutputStream.java → ...try/compressed/ZstdEntryOutputStream.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package ru.olegcherednik.zip4jvm.io.out.entry.encrypted;
+package ru.olegcherednik.zip4jvm.io.out.entry.compressed;
 
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
 import ru.olegcherednik.zip4jvm.io.zstd.ZstdOutputStream;

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/xxx/DataDescriptorOut.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/xxx/DataDescriptorOut.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package ru.olegcherednik.zip4jvm.io.out.entry.xxx;
 
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/xxx/LocalFileHeaderOut.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/entry/xxx/LocalFileHeaderOut.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package ru.olegcherednik.zip4jvm.io.out.entry.xxx;
 
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;