commit ba704f18e5d4bbaf4cfc8c9956c495a62af3a953 Author: Walter Oggioni Date: Sun Apr 26 17:00:10 2020 +0100 initial commit diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..50b3426 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,57 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This generated file contains a sample Kotlin library project to get you started. + */ + +fun property(name : String) = project.property(name).toString() + +plugins { + // Apply the Kotlin JVM plugin to add support for Kotlin. + id("org.jetbrains.kotlin.jvm") version "1.3.70" + + // Apply the java-library plugin for API and implementation separation. + `java-library` + application +} + +repositories { + // Use jcenter for resolving dependencies. + // You can declare any Maven/Ivy/file repository here. + mavenLocal() + jcenter() + mavenCentral() +} + +dependencies { + // Align versions of all Kotlin components + compileOnly(platform("org.jetbrains.kotlin:kotlin-bom")) + testImplementation(platform("org.jetbrains.kotlin:kotlin-bom")) + + // Use the Kotlin JDK 8 standard library. + compileOnly("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + testImplementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + + implementation("net.java.dev.jna", "jna", "5.5.0") + + // https://mvnrepository.com/artifact/org.projectlombok/lombok + compileOnly("org.projectlombok", "lombok", property("lombok.version")) + annotationProcessor("org.projectlombok", "lombok", property("lombok.version")) + + testCompileOnly("org.projectlombok", "lombok", property("lombok.version")) + testAnnotationProcessor("org.projectlombok", "lombok", property("lombok.version")) + testRuntimeOnly("org.junit.jupiter", "junit-jupiter-engine", property("junit.version")) + testImplementation("org.junit.jupiter","junit-jupiter-api", property("junit.version")) + testImplementation("net.woggioni", "jwo", "1.0") + +} + +application { + mainClassName = "net.woggioni.kzstd.HelloWorld" +} + + +tasks.withType { + useJUnitPlatform { + } +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..1760ac5 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,2 @@ +junit.version=5.6.2 +lombok.version=1.18.12 \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..f6cba44 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,10 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/6.3/userguide/multi_project_builds.html + */ + +rootProject.name = "kzstd" diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_CCtx.java b/src/main/java/net/woggioni/jzstd/ZSTD_CCtx.java new file mode 100644 index 0000000..cc0da9a --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_CCtx.java @@ -0,0 +1,5 @@ +package net.woggioni.jzstd; + +import com.sun.jna.PointerType; + +public class ZSTD_CCtx extends PointerType { } diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_CDict.java b/src/main/java/net/woggioni/jzstd/ZSTD_CDict.java new file mode 100644 index 0000000..3ba3c9e --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_CDict.java @@ -0,0 +1,5 @@ +package net.woggioni.jzstd; + +import com.sun.jna.PointerType; + +public class ZSTD_CDict extends PointerType { } diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_EndDirective.java b/src/main/java/net/woggioni/jzstd/ZSTD_EndDirective.java new file mode 100644 index 0000000..bee496f --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_EndDirective.java @@ -0,0 +1,12 @@ +package net.woggioni.jzstd; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ZSTD_EndDirective { + ZSTD_e_continue(0), + ZSTD_e_flush(1), + ZSTD_e_end(2); + + public final int value; +} diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_ResetDirective.java b/src/main/java/net/woggioni/jzstd/ZSTD_ResetDirective.java new file mode 100644 index 0000000..cb0e909 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_ResetDirective.java @@ -0,0 +1,13 @@ +package net.woggioni.jzstd; + + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ZSTD_ResetDirective { + ZSTD_reset_session_only(1), + ZSTD_reset_parameters(2), + ZSTD_reset_session_and_parameters(3); + + public final int value; +} diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_bounds.java b/src/main/java/net/woggioni/jzstd/ZSTD_bounds.java new file mode 100644 index 0000000..85fbaef --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_bounds.java @@ -0,0 +1,14 @@ +package net.woggioni.jzstd; + +import com.sun.jna.Structure; +import net.woggioni.jzstd.internal.size_t; + +@Structure.FieldOrder({"error", "lowerBound", "upperBound"}) +public class ZSTD_bounds extends Structure { + + public size_t error; + public int lowerBound; + public int upperBound; + + public static class ByValue extends ZSTD_bounds implements Structure.ByValue {} +} \ No newline at end of file diff --git a/src/main/java/net/woggioni/jzstd/ZSTD_cParameter.java b/src/main/java/net/woggioni/jzstd/ZSTD_cParameter.java new file mode 100644 index 0000000..e6d0fa7 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZSTD_cParameter.java @@ -0,0 +1,28 @@ +package net.woggioni.jzstd; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ZSTD_cParameter { + compressionLevel(100), + windowLog(101), + hashLog(102), + chainLog(103), + searchLog(104), + minMatch(105), + targetLength(106), + strategy(107), + enableLongDistanceMatching(160), + ldmHashLog(161), + ldmMinMatch(162), + ldmBucketSizeLog(163), + ldmHashRateLog(164), + contentSizeFlag(200), + checksumFlag(201), + dictIDFlag(202), + nbWorkers(400), + jobSize(401), + overlapLog(402); + + public final int value; +} diff --git a/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java b/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java new file mode 100644 index 0000000..4d5974d --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/ZstdOutputStream.java @@ -0,0 +1,124 @@ +package net.woggioni.jzstd; + +import lombok.SneakyThrows; +import net.woggioni.jzstd.internal.ZSTD_inBuffer; +import net.woggioni.jzstd.internal.ZSTD_outBuffer; +import net.woggioni.jzstd.internal.ZstdLibrary; +import net.woggioni.jzstd.internal.size_t; + +import java.io.OutputStream; + +public class ZstdOutputStream extends OutputStream { + private final OutputStream sink; + private final ZSTD_CCtx ctx; + private final boolean ctx_owner; + + private final byte[] buffer; + private final ZSTD_inBuffer input; + private final ZSTD_outBuffer output; + private int pos; + private ZSTD_EndDirective flag; + + private ZstdOutputStream(OutputStream sink, + ZSTD_CCtx ctx, + boolean ctx_owner, + int bufferSize) { + this.sink = sink; + this.ctx = ctx; + this.ctx_owner = ctx_owner; + this.buffer = new byte[bufferSize]; + this.input = new ZSTD_inBuffer(bufferSize); + this.output = new ZSTD_outBuffer(bufferSize); + this.pos = 0; + this.flag = ZSTD_EndDirective.ZSTD_e_continue; + } + + public static ZstdOutputStream from(OutputStream sink, + ZSTD_CCtx ctx) { + return from(sink, ctx, 0x10000); + } + + public static ZstdOutputStream from( + OutputStream sink, + ZSTD_CCtx ctx, + int bufferSize) { + ZstdLibrary.CCtx_reset(ctx, ZSTD_ResetDirective.ZSTD_reset_session_only); + ZstdLibrary.CCtx_refCDict(ctx, null); + return new ZstdOutputStream(sink, ctx, false, bufferSize); + } + + public static ZstdOutputStream from(OutputStream sink) { + return from(sink, 3); + } + + public static ZstdOutputStream from(OutputStream sink, + int compressionLevel) { + return from(sink, compressionLevel, 0x1000); + } + + public static ZstdOutputStream from(OutputStream sink, + int compressionLevel, + int bufferSize) { + ZSTD_bounds range = ZstdLibrary.cParam_getBounds(ZSTD_cParameter.compressionLevel); + if (compressionLevel < range.lowerBound || compressionLevel > range.upperBound) { + throw new IllegalArgumentException("Compression level must be between " + + range.lowerBound + " and " + range.upperBound); + } + ZSTD_CCtx ctx = ZstdLibrary.ZSTD_createCCtx(); + ZstdLibrary.CCtx_setParameter(ctx, + ZSTD_cParameter.compressionLevel, compressionLevel); + return new ZstdOutputStream(sink, ctx, true, bufferSize); + } + + + @Override + @SneakyThrows + public void flush() { + input.src.put(buffer, 0, pos); + input.src.position(0); + input.pos = new size_t(0); + input.size = new size_t(pos); + pos = 0; + while (true) { + size_t rc = ZstdLibrary.compressStream2(ctx, output, input, flag); + if (output.pos.longValue() > 0) { + output.dst.get(buffer); + output.dst.position(0); + sink.write(buffer, 0, output.pos.intValue()); + output.pos = new size_t(0); + } + if (input.pos.longValue() == input.size.longValue() && flag == ZSTD_EndDirective.ZSTD_e_continue) { + break; + } else if (flag == ZSTD_EndDirective.ZSTD_e_end && rc.intValue() == 0) { + break; + } + } + } + + @Override + public void write(int b) { + if (pos == buffer.length) flush(); + buffer[pos++] = (byte) b; + } + + @Override + public void write(byte[] arr, int off, int len) { + var written = 0; + while (written < len - off) { + int writeSize = Math.min(len, buffer.length - pos); + System.arraycopy(arr, off, buffer, pos, writeSize); + pos += writeSize; + if (pos == buffer.length) flush(); + written += writeSize; + } + } + + @Override + @SneakyThrows + public void close() { + flag = ZSTD_EndDirective.ZSTD_e_end; + flush(); + sink.close(); + if (ctx_owner) ZstdLibrary.ZSTD_freeCCtx(ctx); + } +} diff --git a/src/main/java/net/woggioni/jzstd/internal/ZSTD_ErrorCode.java b/src/main/java/net/woggioni/jzstd/internal/ZSTD_ErrorCode.java new file mode 100644 index 0000000..84336cf --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/internal/ZSTD_ErrorCode.java @@ -0,0 +1,32 @@ +package net.woggioni.jzstd.internal; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ZSTD_ErrorCode { + ZSTD_error_no_error(0), + ZSTD_error_GENERIC(1), + ZSTD_error_prefix_unknown(10), + ZSTD_error_version_unsupported(12), + ZSTD_error_frameParameter_unsupported(14), + ZSTD_error_frameParameter_windowTooLarge(16), + ZSTD_error_corruption_detected(20), + ZSTD_error_checksum_wrong(22), + ZSTD_error_dictionary_corrupted(30), + ZSTD_error_dictionary_wrong(32), + ZSTD_error_dictionaryCreation_failed(34), + ZSTD_error_parameter_unsupported(40), + ZSTD_error_parameter_outOfBound(42), + ZSTD_error_tableLog_tooLarge(44), + ZSTD_error_maxSymbolValue_tooLarge(46), + ZSTD_error_maxSymbolValue_tooSmall(48), + ZSTD_error_stage_wrong(60), + ZSTD_error_init_missing(62), + ZSTD_error_memory_allocation(64), + ZSTD_error_workSpace_tooSmall(66), + ZSTD_error_dstSize_tooSmall(70), + ZSTD_error_srcSize_wrong(72), + ZSTD_error_dstBuffer_null(74); + + public final int value; +} \ No newline at end of file diff --git a/src/main/java/net/woggioni/jzstd/internal/ZSTD_inBuffer.java b/src/main/java/net/woggioni/jzstd/internal/ZSTD_inBuffer.java new file mode 100644 index 0000000..bb2c501 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/internal/ZSTD_inBuffer.java @@ -0,0 +1,19 @@ +package net.woggioni.jzstd.internal; + +import com.sun.jna.Structure; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +@Structure.FieldOrder(value = {"src", "size", "pos"}) +public class ZSTD_inBuffer extends Structure { + public ByteBuffer src; + public size_t size; + public size_t pos; + + public ZSTD_inBuffer(int size) { + this.src = ByteBuffer.allocateDirect(size).order(ByteOrder.nativeOrder()); + this.size = new size_t(this.src.capacity()); + this.pos = new size_t(this.src.capacity()); + } +} diff --git a/src/main/java/net/woggioni/jzstd/internal/ZSTD_outBuffer.java b/src/main/java/net/woggioni/jzstd/internal/ZSTD_outBuffer.java new file mode 100644 index 0000000..1d432d9 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/internal/ZSTD_outBuffer.java @@ -0,0 +1,19 @@ +package net.woggioni.jzstd.internal; + +import com.sun.jna.Structure; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +@Structure.FieldOrder({"dst", "size", "pos"}) +public class ZSTD_outBuffer extends Structure { + public ByteBuffer dst; + public size_t size; + public size_t pos; + + public ZSTD_outBuffer(int size) { + this.dst = ByteBuffer.allocateDirect(size).order(ByteOrder.nativeOrder()); + this.size = new size_t(dst.capacity()); + this.pos = new size_t(0); + } +} \ No newline at end of file diff --git a/src/main/java/net/woggioni/jzstd/internal/ZstdLibrary.java b/src/main/java/net/woggioni/jzstd/internal/ZstdLibrary.java new file mode 100644 index 0000000..b17e950 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/internal/ZstdLibrary.java @@ -0,0 +1,82 @@ +package net.woggioni.jzstd.internal; + +import com.sun.jna.Native; +import net.woggioni.jzstd.*; + +public class ZstdLibrary { + + public static void checkReturnCode(size_t rc) { + if (ZSTD_isError(rc)) { + String error = getErrorString(rc); + throw new RuntimeException(error); + } + } + + public static size_t compressStream2( + ZSTD_CCtx cctx, + ZSTD_outBuffer output, + ZSTD_inBuffer input, + ZSTD_EndDirective endOp) { + size_t rc = ZSTD_compressStream2(cctx, output, input, endOp.value); + checkReturnCode(rc); + return rc; + } + + public static ZSTD_bounds cParam_getBounds(ZSTD_cParameter cParam) { + ZSTD_bounds bounds = ZSTD_cParam_getBounds(cParam.value); + checkReturnCode(bounds.error); + return bounds; + } + + public static void CCtx_setParameter(ZSTD_CCtx cctx, ZSTD_cParameter param, int value) { + size_t rc = ZSTD_CCtx_setParameter(cctx, param.value, value); + if (ZSTD_isError(rc)) { + throw new RuntimeException(getErrorString(rc)); + } + } + + public static ZSTD_ErrorCode getErrorCode(size_t functionResult) { + int code = ZSTD_getErrorCode(functionResult); + for (ZSTD_ErrorCode result : ZSTD_ErrorCode.values()) { + if (code == result.value) return result; + } + throw new IllegalArgumentException("Unknown error code " + code); + } + + public static String getErrorString(size_t functionResult) { + return ZSTD_getErrorString(getErrorCode(functionResult).value); + } + + public static void CCtx_reset(ZSTD_CCtx ctx, ZSTD_ResetDirective resetDirective) { + checkReturnCode(ZSTD_CCtx_reset(ctx, resetDirective.value)); + } + + public static void CCtx_refCDict(ZSTD_CCtx ctx, ZSTD_CDict cdict) { + checkReturnCode(ZSTD_CCtx_refCDict(ctx, cdict)); + } + + public static native size_t ZSTD_CCtx_setParameter(ZSTD_CCtx cctx, int param, int value); + public static native ZSTD_CCtx ZSTD_createCCtx(); + public static native size_t ZSTD_freeCCtx(ZSTD_CCtx cctx); + public static native size_t ZSTD_initCStream(ZSTD_CCtx ctx, int compressionLevel); + + public static native ZSTD_CCtx ZSTD_createCStream(); + public static native void ZSTD_freeCStream(ZSTD_CCtx ctx); + public static native size_t ZSTD_compressStream2( + ZSTD_CCtx cctx, + ZSTD_outBuffer output, + ZSTD_inBuffer input, + int endOp); + + public static native ZSTD_bounds.ByValue ZSTD_cParam_getBounds(int cParam); + public static native boolean ZSTD_isError(size_t code); + public static native int ZSTD_getErrorCode(size_t functionResult); + public static native String ZSTD_getErrorString(int code); + + public static native size_t ZSTD_CCtx_reset(ZSTD_CCtx ctx, int reset_directive); + public static native size_t ZSTD_CCtx_refCDict(ZSTD_CCtx ctx, ZSTD_CDict cdict); + + static { + Native.register("zstd"); + } +} diff --git a/src/main/java/net/woggioni/jzstd/internal/size_t.java b/src/main/java/net/woggioni/jzstd/internal/size_t.java new file mode 100644 index 0000000..9097690 --- /dev/null +++ b/src/main/java/net/woggioni/jzstd/internal/size_t.java @@ -0,0 +1,11 @@ +package net.woggioni.jzstd.internal; + +import com.sun.jna.IntegerType; +import com.sun.jna.Native; + +public class size_t extends IntegerType { + private static final long serialVersionUID = 1L; + + public size_t() { this(0); } + public size_t(long value) { super(Native.SIZE_T_SIZE, value); } +} \ No newline at end of file diff --git a/src/main/kotlin/net/woggioni/kzstd/Library.kt b/src/main/kotlin/net/woggioni/kzstd/Library.kt new file mode 100644 index 0000000..2ce6941 --- /dev/null +++ b/src/main/kotlin/net/woggioni/kzstd/Library.kt @@ -0,0 +1,336 @@ +/* + * This Kotlin source file was generated by the Gradle 'init' task. + */ +package net.woggioni.kzstd + +import com.sun.jna.* +import java.io.BufferedInputStream +import java.io.InputStream +import java.io.OutputStream +import java.lang.IllegalArgumentException +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.Paths +import kotlin.math.min + +import net.woggioni.jzstd.internal.ZSTD_inBuffer as jZSTD_inBuffer +import net.woggioni.jzstd.internal.ZSTD_outBuffer as jZSTD_outBuffer +import net.woggioni.jzstd.ZSTD_CCtx as jZSTD_CCtx + +class ZstdInputStream private constructor( + private val source: InputStream, + private val ctx: ZstdLibrary.ZSTD_CCtx, + private val ctx_owner: Boolean, + bufferSize: Int) : InputStream() { + + override fun read(): Int { + TODO("Not yet implemented") + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + return super.read(b, off, len) + } +} + +class ZstdOutputStream private constructor( + private val sink: OutputStream, + private val ctx: ZstdLibrary.ZSTD_CCtx, + private val ctx_owner: Boolean, + bufferSize: Int) : OutputStream() { + + companion object { + + @JvmStatic + @JvmOverloads + fun from(sink: OutputStream, + ctx: ZstdLibrary.ZSTD_CCtx, + bufferSize: Int = 0x10000) = let { + ZstdLibrary.CCtx_reset(ctx, ZstdLibrary.ZSTD_ResetDirective.ZSTD_reset_session_only) + ZstdLibrary.CCtx_refCDict(ctx, null) + ZstdOutputStream(sink, ctx, false, bufferSize) + } + + @JvmStatic + @JvmOverloads + fun from(sink: OutputStream, + compressionLevel: Int = 3, + bufferSize: Int = 0x10000) = let { + val range = ZstdLibrary.ZSTD_cParam_getBounds( + ZstdLibrary.ZSTD_cParameter.compressionLevel.value) + if (compressionLevel < range.lowerBound || compressionLevel > range.upperBound) { + throw IllegalArgumentException("Compression level must be between " + + "${range.lowerBound} and ${range.upperBound}") + } + val ctx = ZstdLibrary.ZSTD_createCCtx() + ZstdLibrary.CCtx_setParameter(ctx, + ZstdLibrary.ZSTD_cParameter.compressionLevel, compressionLevel) + ZstdOutputStream(sink, ctx, true, bufferSize) + } + } + + private val buffer = ByteArray(bufferSize) + private val input = ZstdLibrary.ZSTD_inBuffer(bufferSize) + private val output = ZstdLibrary.ZSTD_outBuffer(bufferSize) + private var pos = 0 + private var flag = ZstdLibrary.ZSTD_EndDirective.ZSTD_e_continue + + + override fun flush() { + input.src.put(buffer, 0, pos) + input.src.position(0) + input.pos = 0 + input.size = pos.toLong() + pos = 0 + while (true) { + val rc = ZstdLibrary.compressStream2(ctx, output, input, flag) + if (output.pos > 0) { + output.dst.get(buffer) + output.dst.position(0) + sink.write(buffer, 0, output.pos.toInt()) + output.pos = 0 + } + if (input.pos == input.size && flag == ZstdLibrary.ZSTD_EndDirective.ZSTD_e_continue) { + break + } else if (flag == ZstdLibrary.ZSTD_EndDirective.ZSTD_e_end && rc == 0L) { + break + } + } + } + + override fun write(b: Int) { + if (pos == buffer.size) flush() + buffer[pos++] = b.toByte() + } + + override fun write(b: ByteArray, off: Int, len: Int) { + var written = 0 + while (written < len - off) { + val writeSize = min(len, buffer.size - pos) + System.arraycopy(b, off, buffer, pos, writeSize) + pos += writeSize + if (pos == buffer.size) flush() + written += writeSize + } + } + + override fun close() { + flag = ZstdLibrary.ZSTD_EndDirective.ZSTD_e_end + flush() + sink.close() + if (ctx_owner) ZstdLibrary.ZSTD_freeCCtx(ctx) + } +} + +class HelloWorld { + + inline fun withResource(closeable: T, body: (resource: T) -> R): R { + return closeable.use { + body(it) + } + } + + private fun checkReturnCode(rc: Long) { + if (ZstdLibrary.ZSTD_isError(rc)) { + val error = ZstdLibrary.getErrorString(rc) + throw RuntimeException(error) + } + } + + companion object { + @JvmStatic + fun main(args: Array) { + + BufferedInputStream(HelloWorld::class.java.getResourceAsStream("/Richard_Dawkins_The_Selfish_Gene.pdf")).use { inputStream -> + Paths.get("/tmp/Richard_Dawkins_The_Selfish_Gene.pdf.zst").let { + ZstdOutputStream.from(Files.newOutputStream(it)) + }.use { outputStream -> + val buffer = ByteArray(0x1000) + while (true) { + val read = inputStream.read(buffer) + if (read < 0) break + outputStream.write(buffer, 0, read) + } + } + } + } + } +} + +object ZstdLibrary { + + fun checkReturnCode(rc: Long) { + if (ZSTD_isError(rc)) { + val error = getErrorString(rc) + throw RuntimeException(error) + } + } + + class ZSTD_CCtx : PointerType() + + class ZSTD_CDict : PointerType() + + + @Structure.FieldOrder(value = ["src", "size", "pos"]) + class ZSTD_inBuffer(size: Int = 0) : Structure() { + @JvmField + var src = ByteBuffer.allocateDirect(size) + + @JvmField + var size: Long = src.capacity().toLong() + + @JvmField + var pos: Long = src.capacity().toLong() + } + + @Structure.FieldOrder(value = ["dst", "size", "pos"]) + class ZSTD_outBuffer(size: Int = 0) : Structure() { + @JvmField + var dst = ByteBuffer.allocateDirect(size) + + @JvmField + var size: Long = dst.capacity().toLong() + + @JvmField + var pos: Long = 0 + } + + @Structure.FieldOrder(value = ["error", "lowerBound", "upperBound"]) + open class ZSTD_bounds( + @JvmField + var error: Long = 0, + @JvmField + var lowerBound: Int = 0, + @JvmField + var upperBound: Int = 0 + ) : Structure() { + class ByValue : ZSTD_bounds(), Structure.ByValue + } + + enum class ZSTD_ResetDirective(@JvmField val value: Int) { + ZSTD_reset_session_only(1), + ZSTD_reset_parameters(2), + ZSTD_reset_session_and_parameters(3) + } + + enum class ZSTD_EndDirective(@JvmField val value: Int) { + ZSTD_e_continue(0), + ZSTD_e_flush(1), + ZSTD_e_end(2) + } + + enum class ZSTD_cParameter(@JvmField val value: Int) { + compressionLevel(100), + windowLog(101), + hashLog(102), + chainLog(103), + searchLog(104), + minMatch(105), + targetLength(106), + strategy(107), + enableLongDistanceMatching(160), + ldmHashLog(161), + ldmMinMatch(162), + ldmBucketSizeLog(163), + ldmHashRateLog(164), + contentSizeFlag(200), + checksumFlag(201), + dictIDFlag(202), + nbWorkers(400), + jobSize(401), + overlapLog(402) + } + + enum class ZSTD_ErrorCode(@JvmField val value: Int) { + ZSTD_error_no_error(0), + ZSTD_error_GENERIC(1), + ZSTD_error_prefix_unknown(10), + ZSTD_error_version_unsupported(12), + ZSTD_error_frameParameter_unsupported(14), + ZSTD_error_frameParameter_windowTooLarge(16), + ZSTD_error_corruption_detected(20), + ZSTD_error_checksum_wrong(22), + ZSTD_error_dictionary_corrupted(30), + ZSTD_error_dictionary_wrong(32), + ZSTD_error_dictionaryCreation_failed(34), + ZSTD_error_parameter_unsupported(40), + ZSTD_error_parameter_outOfBound(42), + ZSTD_error_tableLog_tooLarge(44), + ZSTD_error_maxSymbolValue_tooLarge(46), + ZSTD_error_maxSymbolValue_tooSmall(48), + ZSTD_error_stage_wrong(60), + ZSTD_error_init_missing(62), + ZSTD_error_memory_allocation(64), + ZSTD_error_workSpace_tooSmall(66), + ZSTD_error_dstSize_tooSmall(70), + ZSTD_error_srcSize_wrong(72), + ZSTD_error_dstBuffer_null(74), + } + + fun compressStream2( + cctx: ZSTD_CCtx, + output: ZSTD_outBuffer, + input: ZSTD_inBuffer, + endOp: ZSTD_EndDirective): Long { + val rc = ZSTD_compressStream2(cctx, output, input, endOp.value) + checkReturnCode(rc) + return rc + } + + fun cParam_getBounds(cParam: ZSTD_cParameter): ZSTD_bounds = let { + val bounds = ZSTD_cParam_getBounds(cParam.value) + if (ZSTD_isError(bounds.error)) throw RuntimeException() + bounds + } + + fun CCtx_setParameter(cctx: ZSTD_CCtx, param: ZSTD_cParameter, value: Int) { + val rc = ZSTD_CCtx_setParameter(cctx, param.value, value) + if (ZSTD_isError(rc)) { + throw RuntimeException(getErrorString(rc)) + } + } + + fun getErrorCode(functionResult: Long): ZSTD_ErrorCode { + val code = ZSTD_getErrorCode(functionResult) + for (result in ZSTD_ErrorCode.values()) { + if (code == result.value) return result + } + throw IllegalArgumentException("Unknown error code $code") + } + + fun getErrorString(functionResult: Long): String { + return ZSTD_getErrorString(getErrorCode(functionResult).value) + } + + fun CCtx_reset(ctx: ZSTD_CCtx, resetDirective: ZSTD_ResetDirective) { + checkReturnCode(ZSTD_CCtx_reset(ctx, resetDirective.value)) + } + + fun CCtx_refCDict(ctx: ZSTD_CCtx, cdict: ZSTD_CDict?) { + checkReturnCode(ZSTD_CCtx_refCDict(ctx, cdict)) + } + + external fun ZSTD_CCtx_setParameter(cctx: ZSTD_CCtx, param: Int, value: Int): Long + external fun ZSTD_createCCtx(): ZSTD_CCtx + external fun ZSTD_freeCCtx(cctx: ZSTD_CCtx): Long + external fun ZSTD_initCStream(ctx: ZSTD_CCtx, compressionLevel: Int): Long + + external fun ZSTD_createCStream(): ZSTD_CCtx + external fun ZSTD_freeCStream(p: ZSTD_CCtx) + external fun ZSTD_compressStream2( + cctx: ZSTD_CCtx, + output: ZSTD_outBuffer, + input: ZSTD_inBuffer, + endOp: Int): Long + + external fun ZSTD_cParam_getBounds(cParam: Int): ZSTD_bounds.ByValue + external fun ZSTD_isError(code: Long): Boolean + external fun ZSTD_getErrorCode(functionResult: Long): Int + external fun ZSTD_getErrorString(code: Int): String + + external fun ZSTD_CCtx_reset(ctx: ZSTD_CCtx, reset_directive: Int): Long + external fun ZSTD_CCtx_refCDict(ctx: ZSTD_CCtx, cdict: ZSTD_CDict?): Long + + init { + Native.register("zstd") + } +} \ No newline at end of file diff --git a/src/test/java/net/woggioni/jzstd/SillyTest.java b/src/test/java/net/woggioni/jzstd/SillyTest.java new file mode 100644 index 0000000..54a3f45 --- /dev/null +++ b/src/test/java/net/woggioni/jzstd/SillyTest.java @@ -0,0 +1,61 @@ +package net.woggioni.jzstd; + +import lombok.SneakyThrows; +import net.woggioni.jwo.Chronometer; +import net.woggioni.jzstd.internal.ZstdLibrary; +import org.junit.jupiter.api.Test; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + + +public class SillyTest { + + @Test + @SneakyThrows + public void test() { + var ctx = ZstdLibrary.ZSTD_createCCtx(); + ZstdLibrary.CCtx_setParameter(ctx, ZSTD_cParameter.compressionLevel, 3); + try (InputStream is = new BufferedInputStream( + getClass().getResourceAsStream("/Richard_Dawkins_The_Selfish_Gene.pdf"))) { + try (OutputStream os = + ZstdOutputStream.from( + Files.newOutputStream( + Paths.get("/tmp/Richard_Dawkins_The_Selfish_Gene.pdf.zst")))) { + byte[] buffer = new byte[0x1000]; + while (true) { + int read = is.read(buffer); + if (read < 0) break; + os.write(buffer, 0, read); + } + } + } + } + + @Test + @SneakyThrows + public void test4() { + Chronometer chronometer = new Chronometer(); + Path file = Paths.get("/tmp/citylots.json"); + Path destination = file.getParent().resolve(file.getFileName().toString() + ".zst"); + try (InputStream is = new BufferedInputStream(Files.newInputStream(file))) { + try (OutputStream os = + ZstdOutputStream.from( + Files.newOutputStream(destination), 11)) { + chronometer.reset(); + byte[] buffer = new byte[0x1000]; + while (true) { + int read = is.read(buffer); + if (read < 0) break; + os.write(buffer, 0, read); + } + } + } + System.out.println( + String.format("Elapsed time: %.3f s", chronometer.elapsed(Chronometer.UnitOfMeasure.SECONDS))); + } +} diff --git a/src/test/kotlin/net/woggioni/kstd/SillyTest.kt b/src/test/kotlin/net/woggioni/kstd/SillyTest.kt new file mode 100644 index 0000000..237fac1 --- /dev/null +++ b/src/test/kotlin/net/woggioni/kstd/SillyTest.kt @@ -0,0 +1,58 @@ +package net.woggioni.kstd + +import lombok.SneakyThrows +import net.woggioni.kzstd.ZstdLibrary +import net.woggioni.kzstd.ZstdOutputStream +import org.junit.jupiter.api.Test +import java.io.BufferedInputStream +import java.nio.file.Files +import java.nio.file.Paths + +import net.woggioni.jzstd.internal.ZstdLibrary as jZstdLibrary +import net.woggioni.jzstd.ZSTD_CCtx +import net.woggioni.jzstd.internal.size_t +import net.woggioni.jzstd.internal.ZSTD_outBuffer as jZSTD_outBuffer +import net.woggioni.jzstd.internal.ZSTD_inBuffer as jZSTD_inBuffer + +class SillyTest { + @Test + @SneakyThrows + fun test() { + BufferedInputStream( + javaClass.getResourceAsStream("/Richard_Dawkins_The_Selfish_Gene.pdf")).use { inputstream -> + ZstdOutputStream.from( + Files.newOutputStream( + Paths.get("/tmp/Richard_Dawkins_The_Selfish_Gene.pdf.zst"))).use { outputStream -> + val buffer = ByteArray(0x1000) + while (true) { + val read: Int = inputstream.read(buffer) + if (read < 0) break + outputStream.write(buffer, 0, read) + } + } + } + } + + + @Test + fun test2() { + val ctx = ZstdLibrary.ZSTD_createCCtx() + ZstdLibrary.CCtx_setParameter(ctx, ZstdLibrary.ZSTD_cParameter.compressionLevel, 3) + val input = ZstdLibrary.ZSTD_inBuffer(0x1000) + input.pos = 0 + val output = ZstdLibrary.ZSTD_outBuffer(0x1000) + ZstdLibrary.compressStream2(ctx, output, input, ZstdLibrary.ZSTD_EndDirective.ZSTD_e_continue) + ZstdLibrary.ZSTD_freeCCtx(ctx) + } + + @Test + fun test3() { + val ctx = jZstdLibrary.ZSTD_createCCtx() +// jZstdLibrary.CCtx_setParameter(ctx, ZstdLibrary.ZSTD_cParameter.compressionLevel, 3) + val input = jZSTD_inBuffer(0x1000) + input.pos = size_t(0) + val output = jZSTD_outBuffer(0x1000) +// ZstdLibrary.ZSTD_compressStream2(ctx, output, input, ZstdLibrary.ZSTD_EndDirective.ZSTD_e_continue.value) + jZstdLibrary.ZSTD_freeCCtx(ctx) + } +} \ No newline at end of file diff --git a/src/test/resources/index.html b/src/test/resources/index.html new file mode 100644 index 0000000..e69de29