From 7e08bd7505da746ba03147bb98e363dd0a2dcf39 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Thu, 30 Apr 2020 23:08:49 +0100 Subject: [PATCH] fixed bug --- .../net/woggioni/jzstd/ZstdOutputStream.java | 4 +- .../java/net/woggioni/jzstd/BasicTest.java | 165 +++++++++++++++--- 2 files changed, 144 insertions(+), 25 deletions(-) diff --git a/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java b/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java index e29a53e..3a0ed49 100644 --- a/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java +++ b/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java @@ -89,8 +89,8 @@ public class ZstdOutputStream extends OutputStream { public void write(byte[] arr, int off, int len) { int written = 0; while (written < len - off) { - int writeSize = Math.min(len, input.src.capacity() - input.src.position()); - input.src.put(arr, off, writeSize); + int writeSize = Math.min(len - off - written, input.src.capacity() - input.src.position()); + input.src.put(arr, off + written, writeSize); if (input.src.position() == input.src.capacity()) flush(); written += writeSize; } diff --git a/src/test/java/net/woggioni/jzstd/BasicTest.java b/src/test/java/net/woggioni/jzstd/BasicTest.java index 24b5550..1a72a69 100644 --- a/src/test/java/net/woggioni/jzstd/BasicTest.java +++ b/src/test/java/net/woggioni/jzstd/BasicTest.java @@ -15,11 +15,123 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.util.Random; import java.util.stream.Stream; +class SplitOutputStream extends OutputStream { + private OutputStream[] sinks; + + public SplitOutputStream(OutputStream... sinks) { + this.sinks = sinks; + } + + @Override + public void write(int i) throws IOException { + for (OutputStream os : sinks) { + os.write(i); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + for (OutputStream os : sinks) { + os.write(b, off, len); + } + } + + @Override + public void close() throws IOException { + for (OutputStream os : sinks) { + os.close(); + } + } +} + +class HexSequenceGenerator extends InputStream { + private long count; + private long length; + + private static char[] chars = "0123456789ABCDEF\n".toCharArray(); + + public HexSequenceGenerator(long length) { + this.length = length; + } + + @Override + public int read() { + int result; + if (count < length) { + result = chars[(int)(count % chars.length)]; + ++count; + } else result = -1; + return result; + } + + @Override + @SneakyThrows + public int read(byte[] arr, int off, int len) { + if(count == length) return -1; + else { + int i; + for (i = 0; i < len && count + i < length; i++) { + arr[i] = (byte) chars[(int)((count + i) % chars.length)]; + } + count += i; + return i; + } + } +} + +class RandomBytesGenerator extends InputStream { + private Random random; + + private long count; + private long length; + + public RandomBytesGenerator(long seed, long length) { + this.random = new Random(seed); + this.length = length; + } + + @Override + public int read() { + int result; + if (count < length) { + result = (byte) random.nextInt(); + ++count; + } else result = -1; + return result; + } + + @Override + @SneakyThrows + public int read(byte[] arr, int off, int len) { + if(count == length) return -1; + else { + int i; + for (i = 0; i < len && count + i < length; i++) { + arr[i] = (byte) random.nextInt(); + } + count += i; + return i; + } + } +} public class BasicTest { + final private static char[] hexArray = "0123456789ABCDEF".toCharArray(); + + public static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 2]; + for(int j = 0; j < bytes.length; j++) { + int v = bytes[j] & 0xFF; + hexChars[j * 2] = hexArray[v >>> 4]; + hexChars[j * 2 + 1] = hexArray[v & 0x0F]; + } + return new String(hexChars); + } + private static class InputStreamProvider implements ArgumentsProvider { @Override @@ -27,7 +139,10 @@ public class BasicTest { public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(getClass().getResourceAsStream("/index.html")), - Arguments.of(Files.newInputStream(Paths.get("/tmp/citylots.json"))) + Arguments.of(new RandomBytesGenerator(123456, 1000)), + Arguments.of(new RandomBytesGenerator(654321, 65536)), + Arguments.of(new RandomBytesGenerator(101325, 12345678)), + Arguments.of(new HexSequenceGenerator(12345678)) ); } } @@ -37,47 +152,51 @@ public class BasicTest { @ArgumentsSource(InputStreamProvider.class) public void test(InputStream testStream) { MessageDigest md5 = MessageDigest.getInstance("MD5"); + boolean debug = false; + int compressedSize; ByteArrayInputStream compressedStream; try (InputStream is = new DigestInputStream(new BufferedInputStream(testStream), md5)) { + OutputStream os; ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (OutputStream os = ZstdOutputStream.from(baos)) { - byte[] buffer = new byte[0x1000]; + os = baos; + if(debug) { + os = new SplitOutputStream(os, Files.newOutputStream(Paths.get("/tmp/out.bin.zst"))); + } + os = ZstdOutputStream.from(os); + if (debug) { + new SplitOutputStream( + os, + Files.newOutputStream(Paths.get("/tmp/out.bin.original"))); + } + try { + byte[] buffer = new byte[0x100000]; while (true) { int read = is.read(buffer); if (read < 0) break; os.write(buffer, 0, read); } + } finally { + os.close(); } - compressedStream = new ByteArrayInputStream(baos.toByteArray()); + byte[] compressedBytes = baos.toByteArray(); + compressedSize = compressedBytes.length; + compressedStream = new ByteArrayInputStream(compressedBytes); } + System.out.printf("Compressed size: %d\n", compressedSize); final byte[] originalDigest = md5.digest(); md5.reset(); try (InputStream is = new DigestInputStream( ZstdInputStream.from(compressedStream), md5)) { - byte [] buffer = new byte[0x100000]; - while(true) { + byte[] buffer = new byte[0x10000]; + while (true) { int read = is.read(buffer); - if(read < 0) break; + if (read < 0) break; } } byte[] roundTripDigest = md5.digest(); + System.out.printf("Original digest: %s\n", bytesToHex(originalDigest)); + System.out.printf("Round trip digest: %s\n", bytesToHex(roundTripDigest)); Assertions.assertArrayEquals(originalDigest, roundTripDigest); } - - @SneakyThrows - @Test - public void test2() { - Path inputFile = Paths.get("/tmp/Richard_Dawkins_The_Selfish_Gene.pdf"); - try(InputStream is = Files.newInputStream(inputFile)) { - try(OutputStream os = ZstdOutputStream.from(Files.newOutputStream(Paths.get("/tmp/Richard_Dawkins_The_Selfish_Gene.pdf.zst")))) { - byte[] buffer = new byte[0x10000]; - while(true) { - int read = is.read(buffer); - if(read < 0) break; - os.write(buffer, 0, read); - } - } - } - } }