woggioni/wson
1
0
Fork 0
Code Actions Packages Activity

added binary serialization format

Browse Source
This commit is contained in:
Walter Oggioni
2019-07-16 14:11:23 +02:00
committed by Walter Oggioni
parent 21870cf8a3
commit 2adb6143a5
22 changed files with 875 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
build.sbt
View File

@@ -4,7 +4,6 @@ organization := "org.oggio88"
version := "1.0"
resolvers += Resolver.mavenLocal
scalaVersion := "2.12.6"
scalacOptions ++= Seq(
"-unchecked",
@@ -17,16 +16,16 @@ scalacOptions ++= Seq(
git.useGitDescribe := true
fork := true
//javaOptions in Test += "-Dorg.oggio88.javason.value.ObjectValue.listBasedImplementation=true"
javaOptions in Test += "-Xmx6G"
//javaOptions in Test += "-Dorg.oggio88.worth.value.ObjectValue.listBasedImplementation=true"
//javaOptions in Test += "-Xmx14G"
//scalafmtOnCompile := true
libraryDependencies += "org.projectlombok" % "lombok" % "1.18.2"
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test"
libraryDependencies += "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.6" % "test"
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
libraryDependencies += "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.6" % Test
libraryDependencies += "org.antlr" % "antlr4" % "4.7.1" % "test"
libraryDependencies += "org.antlr" % "antlr4-runtime" % "4.7.1" % "test"
libraryDependencies += "org.tukaani" % "xz" % "1.8" % "test"
libraryDependencies += "org.antlr" % "antlr4" % "4.7.1" % Test
libraryDependencies += "org.antlr" % "antlr4-runtime" % "4.7.1" % Test
libraryDependencies += "org.tukaani" % "xz" % "1.8" % Test
artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
artifact.name + "-" + module.revision + "." + artifact.extension

4
src/main/java/org/oggio88/worth/buffer/LookAheadInputStream.java
View File

@@ -9,7 +9,7 @@ public class LookAheadInputStream extends InputStream {
private final InputStream stream;
private int currentByte;
LookAheadInputStream(InputStream stream) {
public LookAheadInputStream(InputStream stream) {
this.stream = stream;
}
@@ -21,7 +21,7 @@ public class LookAheadInputStream extends InputStream {
return result;
}
public int getCurrentByte(){
public int getCurrentByte() {
return currentByte;
}
}

10
src/main/java/org/oggio88/worth/serialization/ValueDumper.java
View File

@@ -11,9 +11,9 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.Map;
import java.util.Stack;
public abstract class ValueDumper implements Dumper {
@@ -62,10 +62,10 @@ public abstract class ValueDumper implements Dumper {
}
protected Stack<StackLevel> stack;
protected ArrayDeque<StackLevel> stack;
protected ValueDumper() {
stack = new Stack<>();
stack = new ArrayDeque<>();
}
@Override
@@ -83,11 +83,11 @@ public abstract class ValueDumper implements Dumper {
dump(value, new OutputStreamWriter(stream, encoding));
}
protected abstract void beginObject();
protected abstract void beginObject(int size);
protected abstract void endObject();
protected abstract void beginArray();
protected abstract void beginArray(int size);
protected abstract void endArray();

38
src/main/java/org/oggio88/worth/serialization/ValueParser.java
View File

@@ -3,12 +3,7 @@ package org.oggio88.worth.serialization;
import lombok.RequiredArgsConstructor;
import org.oggio88.worth.exception.NotImplementedException;
import org.oggio88.worth.utils.WorthUtils;
import org.oggio88.worth.value.ArrayValue;
import org.oggio88.worth.value.BooleanValue;
import org.oggio88.worth.value.FloatValue;
import org.oggio88.worth.value.IntegerValue;
import org.oggio88.worth.value.ObjectValue;
import org.oggio88.worth.value.StringValue;
import org.oggio88.worth.value.*;
import org.oggio88.worth.xface.Parser;
import org.oggio88.worth.xface.Value;
@@ -16,18 +11,22 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.Charset;
import java.util.Stack;
import java.util.ArrayDeque;
public class ValueParser implements Parser {
@RequiredArgsConstructor
protected static class StackLevel {
public final Value value;
public final long expectedSize;
}
protected static class ArrayStackLevel extends StackLevel {
public ArrayStackLevel() {
super(new ArrayValue());
super(new ArrayValue(), -1);
}
public ArrayStackLevel(long expectedSize) {
super(new ArrayValue(), expectedSize);
}
}
@@ -35,14 +34,18 @@ public class ValueParser implements Parser {
public String currentKey;
public ObjectStackLevel() {
super(ObjectValue.newInstance());
super(ObjectValue.newInstance(), -1);
}
public ObjectStackLevel(long expectedSize) {
super(ObjectValue.newInstance(), expectedSize);
}
}
protected Stack<StackLevel> stack;
protected ArrayDeque<StackLevel> stack;
private void add2Last(Value value) {
StackLevel last = stack.lastElement();
StackLevel last = stack.getFirst();
ArrayStackLevel asl;
ObjectStackLevel osl;
if ((asl = WorthUtils.dynamicCast(last, ArrayStackLevel.class)) != null)
@@ -54,7 +57,7 @@ public class ValueParser implements Parser {
}
protected ValueParser() {
stack = new Stack<>();
stack = new ArrayDeque<>();
stack.push(new ArrayStackLevel());
}
@@ -77,6 +80,11 @@ public class ValueParser implements Parser {
stack.push(new ObjectStackLevel());
}
protected void beginObject(long size) {
stack.push(new ObjectStackLevel(size));
}
protected void endObject() {
add2Last(stack.pop().value);
}
@@ -85,12 +93,16 @@ public class ValueParser implements Parser {
stack.push(new ArrayStackLevel());
}
protected void beginArray(long size) {
stack.push(new ArrayStackLevel(size));
}
protected void endArray() {
add2Last(stack.pop().value);
}
protected void objectKey(String key) {
ObjectStackLevel osl = (ObjectStackLevel) stack.lastElement();
ObjectStackLevel osl = (ObjectStackLevel) stack.getFirst();
osl.currentKey = key;
}

22
src/main/java/org/oggio88/worth/serialization/binary/BinaryMarker.java Normal file
View File

@@ -0,0 +1,22 @@
package org.oggio88.worth.serialization.binary;
public enum BinaryMarker {
Float(0x0),
Int(0x1),
Null(0x2),
True(0x3),
False(0x4),
Reference(0x5),
Id(0x6),
EmptyString(0x0d),
LargeString(0x5d),
EmptyObject(0x5e),
LargeObject(0xae),
EmptyArray(0xaf),
LargeArray(0xff);
public byte value;
BinaryMarker(int b) {
value = (byte) b;
}
}

175
src/main/java/org/oggio88/worth/serialization/binary/JBONDumper.java Normal file
View File

@@ -0,0 +1,175 @@
package org.oggio88.worth.serialization.binary;
import lombok.SneakyThrows;
import org.oggio88.worth.exception.NotImplementedException;
import org.oggio88.worth.serialization.ValueDumper;
import org.oggio88.worth.utils.Leb128;
import org.oggio88.worth.value.ArrayValue;
import org.oggio88.worth.value.ObjectValue;
import org.oggio88.worth.xface.Dumper;
import org.oggio88.worth.xface.Value;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Map;
import java.util.function.Consumer;
import static org.oggio88.worth.utils.WorthUtils.dynamicCast;
public class JBONDumper extends ValueDumper {
public static Dumper newInstance() {
return new JBONDumper();
}
protected OutputStream os;
@Override
public void dump(Value value, Writer writer) {
throw new NotImplementedException(null);
}
@Override
@SneakyThrows
public void dump(Value value, OutputStream outputStream) {
this.os = outputStream;
final Consumer<Value> handle_value = (v) -> {
switch (v.type()) {
case NULL:
nullValue();
break;
case BOOLEAN:
booleanValue(v.asBoolean());
break;
case INTEGER:
integerValue(v.asInteger());
break;
case DOUBLE:
floatValue(v.asFloat());
break;
case STRING:
stringValue(v.asString());
break;
case ARRAY:
ArrayValue arrayValue = dynamicCast(v, ArrayValue.class);
stack.push(new ArrayStackLevel(arrayValue));
beginArray(arrayValue.size());
break;
case OBJECT:
ObjectValue objectValue = dynamicCast(v, ObjectValue.class);
stack.push(new ObjectStackLevel(objectValue));
beginObject(objectValue.size());
break;
}
};
handle_value.accept(value);
while (stack.size() > 0) {
StackLevel last = stack.getFirst();
ArrayStackLevel arrayStackLevel;
ObjectStackLevel objectStackLevel;
if ((arrayStackLevel = dynamicCast(last, ArrayStackLevel.class)) != null) {
if (arrayStackLevel.hasNext()) {
handle_value.accept(arrayStackLevel.next());
} else {
endArray();
stack.pop();
}
} else if ((objectStackLevel = dynamicCast(last, ObjectStackLevel.class)) != null) {
if (objectStackLevel.hasNext()) {
Map.Entry<String, Value> entry = objectStackLevel.next();
objectKey(entry.getKey());
handle_value.accept(entry.getValue());
} else {
endObject();
stack.pop();
}
}
}
this.os.flush();
this.os = null;
}
@Override
@SneakyThrows
protected void beginObject(int size) {
if(size == 0) {
os.write(BinaryMarker.EmptyObject.value);
} else {
os.write(BinaryMarker.LargeObject.value);
Leb128.encode(os, size);
}
}
@Override
@SneakyThrows
protected void endObject() {
}
@Override
@SneakyThrows
protected void beginArray(int size) {
if(size == 0) {
os.write(BinaryMarker.EmptyArray.value);
} else {
os.write(BinaryMarker.LargeArray.value);
Leb128.encode(os, size);
}
}
@Override
@SneakyThrows
protected void endArray() {
}
@Override
@SneakyThrows
protected void objectKey(String key) {
byte[] bytes = key.getBytes();
Leb128.encode(os, bytes.length);
os.write(key.getBytes());
}
@Override
@SneakyThrows
protected void stringValue(String value) {
if(value.isEmpty()) {
os.write(BinaryMarker.EmptyString.value);
} else {
os.write(BinaryMarker.LargeString.value);
byte[] bytes = value.getBytes();
Leb128.encode(os, bytes.length);
os.write(value.getBytes());
}
}
@Override
@SneakyThrows
protected void integerValue(long value) {
os.write(BinaryMarker.Int.value);
Leb128.encode(os, value);
}
@Override
@SneakyThrows
protected void floatValue(double value) {
os.write(BinaryMarker.Float.value);
Leb128.encode(os, value);
}
@Override
@SneakyThrows
protected void booleanValue(boolean value) {
if(value) {
os.write(BinaryMarker.True.value);
} else {
os.write(BinaryMarker.False.value);
}
}
@Override
@SneakyThrows
protected void nullValue() {
os.write(BinaryMarker.Null.value);
}
}

124
src/main/java/org/oggio88/worth/serialization/binary/JBONParser.java Normal file
View File

@@ -0,0 +1,124 @@
package org.oggio88.worth.serialization.binary;
import lombok.SneakyThrows;
import org.oggio88.worth.buffer.LookAheadInputStream;
import org.oggio88.worth.exception.ParseException;
import org.oggio88.worth.serialization.ValueParser;
import org.oggio88.worth.utils.Leb128;
import org.oggio88.worth.utils.WorthUtils;
import org.oggio88.worth.xface.Parser;
import org.oggio88.worth.xface.Value;
import java.io.InputStream;
import java.util.function.Function;
public class JBONParser extends ValueParser {
private int cursor = 0;
private <T extends RuntimeException> T error(Function<String, T> constructor, String fmt, Object... args) {
return constructor.apply(
String.format("Error at position %d: %s",
cursor, String.format(fmt, args)));
}
@Override
@SneakyThrows
public Value parse(InputStream is) {
final LookAheadInputStream stream = new LookAheadInputStream(is) {
@Override
public int read() {
int result = super.read();
++cursor;
return result;
}
};
stream.read();
try {
Leb128.Leb128Decoder decoder = new Leb128.Leb128Decoder(stream);
ObjectStackLevel osl;
while (true) {
if ((osl = WorthUtils.dynamicCast(stack.getFirst(), ObjectStackLevel.class)) != null && osl.currentKey == null) {
int size = (int) decoder.decode();
byte[] buffer = new byte[size];
stream.read(buffer);
String text = new String(buffer);
objectKey(text);
}
byte b;
{
int c = stream.read();
if(c == -1) {
break;
}
b = (byte) c;
}
if(b == BinaryMarker.Null.value) {
nullValue();
} else if(b == BinaryMarker.True.value) {
booleanValue(true);
} else if(b == BinaryMarker.False.value) {
booleanValue(false);
} else if(b == BinaryMarker.Int.value) {
integerValue(decoder.decode());
} else if(b == BinaryMarker.Float.value) {
floatValue(decoder.decodeDouble());
} else if(b == BinaryMarker.EmptyString.value) {
stringValue("");
} else if(b == BinaryMarker.LargeString.value) {
byte[] buffer = new byte[(int) decoder.decode()];
stream.read(buffer);
String text = new String(buffer);
stringValue(text);
} else if(b == BinaryMarker.EmptyArray.value) {
beginArray(0);
} else if(b == BinaryMarker.LargeArray.value) {
long size = decoder.decode();
beginArray(size);
} else if(b == BinaryMarker.EmptyObject.value) {
beginObject(0);
} else if(b == BinaryMarker.LargeObject.value) {
long size = decoder.decode();
beginObject(size);
} else {
throw new ParseException(String.format("Illegal byte at position %d: 0x%02x", cursor, b));
}
while(stack.size() > 0) {
if ((osl = WorthUtils.dynamicCast(stack.getFirst(), ObjectStackLevel.class)) != null
&& osl.value.size() == osl.expectedSize) {
endObject();
continue;
}
ArrayStackLevel asl;
if((asl = WorthUtils.dynamicCast(stack.getFirst(), ArrayStackLevel.class)) != null
&& asl.value.size() == asl.expectedSize) {
endArray();
continue;
}
break;
}
}
if (stack.size() > 1) {
ValueParser.StackLevel last = stack.getFirst();
String type;
if(last instanceof ArrayStackLevel) {
type = "array";
} else {
type = "object";
}
throw error(ParseException::new, "Unfinished %s", type);
}
return WorthUtils.dynamicCast(stack.getFirst(), ArrayStackLevel.class).value.get(0);
} catch (NumberFormatException | NegativeArraySizeException e) {
throw error(ParseException::new, e.getMessage());
} finally {
stack.clear();
}
}
public static Parser newInstance() {
return new JBONParser();
}
}

16
src/main/java/org/oggio88/worth/serialization/json/JSONDumper.java
View File

@@ -21,7 +21,7 @@ public class JSONDumper extends ValueDumper {
return new JSONDumper();
}
protected Writer writer;
private Writer writer;
private String escapeString(String value){
StringBuilder sb = new StringBuilder();
@@ -81,19 +81,21 @@ public class JSONDumper extends ValueDumper {
stringValue(v.asString());
break;
case ARRAY:
stack.push(new ArrayStackLevel(dynamicCast(v, ArrayValue.class)));
beginArray();
ArrayValue arrayValue = dynamicCast(v, ArrayValue.class);
stack.push(new ArrayStackLevel(arrayValue));
beginArray(arrayValue.size());
break;
case OBJECT:
ObjectValue objectValue = dynamicCast(v, ObjectValue.class);
stack.push(new ObjectStackLevel(dynamicCast(v, ObjectValue.class)));
beginObject();
beginObject(objectValue.size());
break;
}
};
handle_value.accept(value);
while (stack.size() > 0) {
StackLevel last = stack.lastElement();
StackLevel last = stack.getFirst();
ArrayStackLevel arrayStackLevel;
ObjectStackLevel objectStackLevel;
if ((arrayStackLevel = dynamicCast(last, ArrayStackLevel.class)) != null) {
@@ -127,7 +129,7 @@ public class JSONDumper extends ValueDumper {
@Override
@SneakyThrows
protected void beginObject() {
protected void beginObject(int size) {
this.writer.write("{");
}
@@ -139,7 +141,7 @@ public class JSONDumper extends ValueDumper {
@Override
@SneakyThrows
protected void beginArray() {
protected void beginArray(int size) {
this.writer.write("[");
}

14
src/main/java/org/oggio88/worth/serialization/json/JSONParser.java
View File

@@ -69,7 +69,7 @@ public class JSONParser extends ValueParser {
}
}
private final String readString(LookAheadTextInputStream stream) {
private String readString(LookAheadTextInputStream stream) {
StringBuilder sb = new StringBuilder();
boolean escape = false;
boolean start = false;
@@ -116,7 +116,7 @@ public class JSONParser extends ValueParser {
return sb.toString();
}
private final void consumeExpected(LookAheadTextInputStream stream, String expected, String errorMessage) {
private void consumeExpected(LookAheadTextInputStream stream, String expected, String errorMessage) {
int i = 0;
while (true) {
int c = stream.getCurrentByte();
@@ -185,11 +185,9 @@ public class JSONParser extends ValueParser {
throw error(ParseException::new, nfe.getMessage());
}
} else if (c == '\"') {
if (currentLine == 125)
System.out.print("");
String text = readString(stream);
ObjectStackLevel osl;
if ((osl = WorthUtils.dynamicCast(stack.lastElement(), ObjectStackLevel.class)) != null && osl.currentKey == null) {
if ((osl = WorthUtils.dynamicCast(stack.getFirst(), ObjectStackLevel.class)) != null && osl.currentKey == null) {
objectKey(text);
} else {
stringValue(text);
@@ -208,16 +206,16 @@ public class JSONParser extends ValueParser {
}
if (stack.size() > 1) {
char c;
if (stack.lastElement() instanceof ArrayStackLevel) {
if (stack.getFirst() instanceof ArrayStackLevel) {
c = ']';
} else if (stack.lastElement() instanceof ObjectStackLevel) {
} else if (stack.getFirst() instanceof ObjectStackLevel) {
c = '}';
} else {
throw new NotImplementedException("This should never happen");
}
throw error(ParseException::new, "Missing '%c' token", c);
}
return WorthUtils.dynamicCast(stack.lastElement(), ArrayStackLevel.class).value.get(0);
return WorthUtils.dynamicCast(stack.getFirst(), ArrayStackLevel.class).value.get(0);
} catch (NumberFormatException e) {
throw error(ParseException::new, e.getMessage());
} finally {

77
src/main/java/org/oggio88/worth/utils/Leb128.java Normal file
View File

@@ -0,0 +1,77 @@
package org.oggio88.worth.utils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import java.io.InputStream;
import java.io.OutputStream;
public class Leb128 {
public static long reverse(long n) {
long res = 0;
for(int i = 0; i < 8; i++) {
long b = (n & (0xFFL << (i * 8))) >>> (i * 8);
res |= b << ((7 - i) * 8);
}
return res;
}
public static int encode(OutputStream os, double input) {
return encode(os, reverse(Double.doubleToLongBits(input)));
}
@SneakyThrows
public static int encode(OutputStream os, long input) {
int bytes_written = 0;
long number = input >= 0 ? (input << 1) : (-(input + 1)) << 1 | 1;
while(number > 127) {
os.write((int) (number & 127) | 128);
bytes_written++;
number >>= 7;
}
os.write((int) number);
return ++bytes_written;
}
@RequiredArgsConstructor
public static class Leb128Decoder {
@Getter
private int bytesRead = 0;
private final InputStream is;
public byte decodeByte() {
return (byte) decode();
}
public short decodeShort() {
return (short) decode();
}
public int decodeInt() {
return (int) decode();
}
public double decodeDouble() {
return Double.longBitsToDouble(reverse(decode()));
}
@SneakyThrows
public long decode() {
long res = 0;
for(int i = 0; i < (8 * 8 + 6) / 7; i++) {
int c = is.read();
bytesRead++;
if(c < 0) {
throw new IllegalArgumentException("Unexpected end of file");
}
byte b = (byte) c;
res |= ((long)(b & 127)) << (i * 7);
if(b >= 0) break;
}
return (res & 1) != 0 ? - (res >> 1) - 1 : (res >> 1);
}
}
}

61
src/main/java/org/oggio88/worth/utils/ValueWalker.java Normal file
View File

@@ -0,0 +1,61 @@
package org.oggio88.worth.utils;
import org.oggio88.worth.xface.Value;
import java.util.Optional;
import java.util.function.Function;
public class ValueWalker {
private Value parent;
public ValueWalker(Value root) {
parent = root;
}
public ValueWalker get(String key) {
if(parent.type() == Value.Type.OBJECT) {
parent = parent.get(key);
} else {
parent = Value.Null;
}
return this;
}
public ValueWalker get(int index) {
if(parent.type() == Value.Type.ARRAY) {
parent = parent.get(index);
} else {
parent = Value.Null;
}
return this;
}
public Value get() {
return parent;
}
public <T> Optional<T> map(Function<Value, T> callback) {
if(isPresent()) {
return Optional.of(callback.apply(parent));
} else {
return Optional.empty();
}
}
public <T> Optional<T> flatMap(Function<Value, Optional<T>> callback) {
if(isPresent()) {
return callback.apply(parent);
} else {
return Optional.empty();
}
}
public boolean isPresent() {
return !isEmpty();
}
public boolean isEmpty() {
return parent.type() == Value.Type.NULL;
}
}

129
src/main/java/org/oggio88/worth/utils/WorthUtils.java
View File

@@ -1,19 +1,20 @@
package org.oggio88.worth.utils;
import lombok.SneakyThrows;
import org.oggio88.worth.buffer.CircularBuffer;
import org.oggio88.worth.exception.ParseException;
import org.oggio88.worth.xface.Value;
import java.io.InputStream;
import java.io.*;
import java.lang.reflect.Constructor;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public class WorthUtils {
@SneakyThrows
public static <T> T uncheckCall(final Callable<T> callable) {
return callable.call();
}
public static <T> T dynamicCast(final Object o, final Class<T> cls) {
if (cls.isInstance(o)) {
return (T) o;
@@ -22,8 +23,114 @@ public class WorthUtils {
}
}
public static boolean equalsNullSafe(Object o1, Object o2) {
if (o1 == null) return o2 == null;
else return o1.equals(o2);
public static <T> Stream<T> iterable2stream(Iterable<T> iterable) {
return StreamSupport.stream(iterable.spliterator(), false);
}
public static void writeObject2File(String fileName, Object o) {
writeObject2File(new File(fileName), o);
}
@SneakyThrows
public static void writeObject2File(File file, Object o) {
try (Writer writer = new OutputStreamWriter(new FileOutputStream(file.getPath()))) {
writer.write(o.toString());
}
}
@SneakyThrows
public static String readFile2String(File file) {
StringBuilder builder = new StringBuilder();
try (Reader reader = new InputStreamReader(new BufferedInputStream(new FileInputStream(file.getPath())))) {
char[] buffer = new char[1024];
while (true) {
int read = reader.read(buffer);
builder.append(buffer, 0, read);
if (read < buffer.length) break;
}
}
return builder.toString();
}
@SneakyThrows
public static String readResource2String(String classpath) {
StringBuilder sb = new StringBuilder();
try (Reader reader = new InputStreamReader(WorthUtils.class.getResourceAsStream(classpath))) {
char[] buffer = new char[1024];
while (true) {
int read = reader.read(buffer);
sb.append(buffer, 0, read);
if (read < buffer.length) break;
}
}
return sb.toString();
}
@SneakyThrows
public static <T extends Throwable> T newThrowable(Class<T> cls, String format, Object... args) {
Constructor<T> constructor = cls.getConstructor(String.class);
return constructor.newInstance(String.format(format, args));
}
@SneakyThrows
public static <T extends Throwable> T newThrowable(Class<T> cls, Throwable throwable, String format, Object... args) {
Constructor<T> constructor = cls.getConstructor(String.class, Throwable.class);
return constructor.newInstance(String.format(format, args), throwable);
}
@SneakyThrows
public static <T extends Throwable> void raise(Class<T> cls, Throwable throwable, String format, Object... args) {
throw newThrowable(cls, throwable, format, args);
}
@SneakyThrows
public static <T extends Throwable> void raise(Class<T> cls, String format, Object... args) {
throw newThrowable(cls, format, args);
}
public static Value getOrNull(Value root, String... keys) {
Value result = root;
for (String key : keys) {
result = result.get(key);
if (result == null) {
result = Value.Null;
break;
}
}
return result;
}
public static <T> T getOrNull(Value root, Function<Value, T> callback, String... keys) {
Value result = getOrNull(root, keys);
return result.type() == Value.Type.NULL ? null : callback.apply(result);
}
@SneakyThrows
public static <T> T getOrThrow(Value root, Function<Value, T> success, Supplier<Throwable> error, String... keys) {
Value result = getOrNull(root, keys);
if (result.type() == Value.Type.NULL) {
throw error.get();
} else {
return success.apply(result);
}
}
public static <T> Predicate<T> not(Predicate<T> p) {
return p.negate();
}
public static <V, T> Stream<V> flatMap(Stream<T> stream,
Function<? super T, Optional<? extends V>> mappingFunction) {
return stream.map(mappingFunction).filter(Optional::isPresent).map(Optional::get);
}
public static <T> Stream<T> optional2Stream(Optional<T> optional) {
return optional.map(Stream::of).orElse(Stream.empty());
}
public static void setSystemPropertyIfNotDefined(String key, String value) {
if(System.getProperty(key) == null) {
System.setProperty(key, value);
}
}
}

9
src/main/java/org/oggio88/worth/value/ArrayValue.java
View File

@@ -6,6 +6,7 @@ import org.oggio88.worth.xface.Value;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
@EqualsAndHashCode
public class ArrayValue implements Value, Iterable<Value> {
@@ -32,7 +33,12 @@ public class ArrayValue implements Value, Iterable<Value> {
@Override
public Value get(int index) {
return value.get(index);
int sz = size();
if(index < sz) {
return value.get(Math.floorMod(index, sz));
} else {
return Value.Null;
}
}
@Override
@@ -63,6 +69,7 @@ public class ArrayValue implements Value, Iterable<Value> {
return value.iterator();
}
@Override
public int size() {
return value.size();
}

49
src/main/java/org/oggio88/worth/value/ObjectValue.java
View File

@@ -6,19 +6,16 @@ import org.oggio88.worth.xface.Value;
import java.util.*;
import static org.oggio88.worth.utils.WorthUtils.equalsNullSafe;
public interface ObjectValue extends Value, Iterable<Map.Entry<String, Value>> {
boolean listBasedImplementation = Boolean.valueOf(
System.getProperty("org.oggio88.javason.value.ObjectValue.listBasedImplementation", "false"));
System.getProperty(ObjectValue.class.getName() + ".listBasedImplementation", "false"));
boolean preserveKeyOrder = Boolean.valueOf(
System.getProperty("org.oggio88.javason.value.MapObjectValue.preserveKeyOrder", "false"));
System.getProperty(ObjectValue.class.getName() + ".preserveKeyOrder", "false"));
static ObjectValue newInstance() {
if (listBasedImplementation) {
return new MapObjectValue();
return new ListObjectValue();
} else {
return new MapObjectValue();
}
@@ -63,7 +60,7 @@ class MapObjectValue implements ObjectValue {
private final Map<String, Value> value;
public MapObjectValue() {
this.value = ObjectValue.preserveKeyOrder ? new LinkedHashMap() : new HashMap();
this.value = ObjectValue.preserveKeyOrder ? new LinkedHashMap<>() : new HashMap<>();
}
public MapObjectValue(Map<String, Value> value) {
@@ -72,25 +69,17 @@ class MapObjectValue implements ObjectValue {
@Override
public Map<String, Value> asObject() {
return value;
return Collections.unmodifiableMap(value);
}
@Override
public Value get(String key) {
Value result = value.get(key);
if (result == null) {
result = Value.Null;
value.put(key, result);
}
return result;
return value.getOrDefault(key, Value.Null);
}
@Override
public Value getOrDefault(String key, Value defaultValue) {
if (value.containsKey(key))
return value.get(key);
else
return defaultValue;
return value.getOrDefault(key, defaultValue);
}
@Override
@@ -118,13 +107,18 @@ class MapObjectValue implements ObjectValue {
public Iterator<Map.Entry<String, Value>> iterator() {
return value.entrySet().iterator();
}
@Override
public int size() {
return value.size();
}
}
@NoArgsConstructor
@EqualsAndHashCode
class ListObjectValue implements ObjectValue {
private final List<Map.Entry<String, Value>> value = new ArrayList();
private final List<Map.Entry<String, Value>> value = new ArrayList<>();
public ListObjectValue(Map<String, Value> map) {
this.value.addAll(map.entrySet());
@@ -132,17 +126,17 @@ class ListObjectValue implements ObjectValue {
@Override
public Map<String, Value> asObject() {
Map<String, Value> result = preserveKeyOrder ? new LinkedHashMap() : new HashMap();
Map<String, Value> result = preserveKeyOrder ? new LinkedHashMap<>() : new HashMap<>();
for (Map.Entry<String, Value> entry : value) {
result.put(entry.getKey(), entry.getValue());
}
return result;
return Collections.unmodifiableMap(result);
}
@Override
public Value get(String key) {
for (Map.Entry<String, Value> entry : value) {
if(equalsNullSafe(entry.getKey(), key)) return entry.getValue();
if(Objects.equals(entry.getKey(), key)) return entry.getValue();
}
return Value.Null;
}
@@ -150,7 +144,7 @@ class ListObjectValue implements ObjectValue {
@Override
public Value getOrDefault(String key, Value defaultValue) {
for (Map.Entry<String, Value> entry : value) {
if(equalsNullSafe(entry.getKey(), key)) return entry.getValue();
if(Objects.equals(entry.getKey(), key)) return entry.getValue();
}
return defaultValue;
}
@@ -158,7 +152,7 @@ class ListObjectValue implements ObjectValue {
@Override
public Value getOrPut(String key, Value value2Put) {
for (Map.Entry<String, Value> entry : value) {
if(equalsNullSafe(entry.getKey(), key)) return entry.getValue();
if(Objects.equals(entry.getKey(), key)) return entry.getValue();
}
put(key, value2Put);
return value2Put;
@@ -173,7 +167,7 @@ class ListObjectValue implements ObjectValue {
@Override
public boolean has(String key) {
for (Map.Entry<String, Value> entry : value) {
if(equalsNullSafe(entry.getKey(), key)) return true;
if(Objects.equals(entry.getKey(), key)) return true;
}
return false;
}
@@ -182,4 +176,9 @@ class ListObjectValue implements ObjectValue {
public Iterator<Map.Entry<String, Value>> iterator() {
return value.iterator();
}
@Override
public int size() {
return value.size();
}
}

5
src/main/java/org/oggio88/worth/xface/Value.java
View File

@@ -5,6 +5,7 @@ import org.oggio88.worth.value.NullValue;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public interface Value {
@@ -40,6 +41,10 @@ public interface Value {
throw new TypeException("Not an object");
}
default int size() {
throw new TypeException("Neither an array nor an object");
}
default void add(Value value) {
throw new TypeException("Not an array");
}

2
src/test/java/org/oggio88/worth/antlr/JSONListenerImpl.java
View File

@@ -20,7 +20,7 @@ public class JSONListenerImpl extends ValueParser implements JSONListener {
@Override
public void exitJson(JSONParser.JsonContext ctx) {
result = stack.get(0).value;
result = stack.getFirst().value;
stack.clear();
}

7
src/test/java/org/oggio88/worth/antlr/ParseTest.java
View File

@@ -1,20 +1,23 @@
package org.oggio88.worth.antlr;
import lombok.SneakyThrows;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Test;
import org.oggio88.worth.serialization.json.JSONDumper;
import org.oggio88.worth.xface.Value;
import java.io.InputStreamReader;
public class ParseTest {
@Test
@SneakyThrows
public void test(){
ANTLRInputStream inputStream = new ANTLRInputStream(getClass().getResourceAsStream("/test.json"));
CodePointCharStream inputStream = CharStreams.fromReader(new InputStreamReader(getClass().getResourceAsStream("/test.json")));
JSONLexer lexer = new JSONLexer(inputStream);
CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
JSONParser parser = new JSONParser(commonTokenStream);

92
src/test/java/org/oggio88/worth/serialization/binary/JBONTest.java Normal file
View File

@@ -0,0 +1,92 @@
package org.oggio88.worth.serialization.binary;
import lombok.SneakyThrows;
import org.junit.Assert;
import org.junit.Test;
import org.oggio88.worth.buffer.LookAheadTextInputStream;
import org.oggio88.worth.serialization.binary.JBONDumper;
import org.oggio88.worth.serialization.binary.JBONParser;
import org.oggio88.worth.serialization.json.JSONParser;
import org.oggio88.worth.value.NullValue;
import org.oggio88.worth.value.ObjectValue;
import org.oggio88.worth.xface.Parser;
import org.oggio88.worth.xface.Value;
import java.io.*;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
public class JBONTest {
private String[] testFiles = new String[]{"/test.json", "/wordpress.json"};
private InputStream getTestSource(String filename) {
return getClass().getResourceAsStream(filename);
}
@Test
@SneakyThrows
public void consistencyTest() {
System.setProperty(ObjectValue.class.getName() + ".preserveKeyOrder", "true");
for (String testFile : testFiles) {
Value parsedValue;
try(InputStream is = getTestSource(testFile)) {
Parser parser = new JSONParser();
parsedValue = parser.parse(is);
}
byte[] dumpedJBON;
try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
JBONDumper.newInstance().dump(parsedValue, baos);
dumpedJBON = baos.toByteArray();
}
Value reParsedValue;
try(InputStream is = new ByteArrayInputStream(dumpedJBON)) {
Parser parser = new JBONParser();
reParsedValue = parser.parse(is);
}
Assert.assertEquals(parsedValue, reParsedValue);
byte[] reDumpedJBON;
try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
JBONDumper.newInstance().dump(reParsedValue, baos);
reDumpedJBON = baos.toByteArray();
}
Assert.assertArrayEquals(dumpedJBON, reDumpedJBON);
}
}
@Test
@SneakyThrows
public void comparativeTest() {
for (String testFile : testFiles) {
Value originalValue = new JSONParser().parse(getTestSource(testFile));
Path outputFile = Files.createTempFile(Paths.get("/tmp"),"worh", null);
try (OutputStream os = new FileOutputStream(outputFile.toFile())) {
JBONDumper jbonDumper = new JBONDumper();
jbonDumper.dump(originalValue, os);
}
Value binarySerializedValue;
try(InputStream is = new FileInputStream(outputFile.toFile())) {
JBONParser jbonParser = new JBONParser();
binarySerializedValue = jbonParser.parse(is);
}
Assert.assertEquals(originalValue, binarySerializedValue);
}
}
@Test
@SneakyThrows
public void hexTest() {
String hex = "1F608";
byte[] buffer = hex.getBytes();
ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
Method method = JSONParser.class.getDeclaredMethod("parseHex", LookAheadTextInputStream.class);
method.setAccessible(true);
LookAheadTextInputStream ltis = new LookAheadTextInputStream(new InputStreamReader(bais));
ltis.read();
int result = (int) method.invoke(null, ltis);
Assert.assertEquals((int) Integer.valueOf(hex, 16), result);
}
}

7
src/test/java/org/oggio88/worth/serialization/json/JSONTest.java
View File

@@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.JsonNodeType;
import lombok.SneakyThrows;
import org.junit.Assert;
import org.junit.Test;
import org.oggio88.worth.buffer.CircularBuffer;
import org.oggio88.worth.buffer.LookAheadTextInputStream;
import org.oggio88.worth.exception.NotImplementedException;
import org.oggio88.worth.utils.WorthUtils;
@@ -187,7 +186,7 @@ public class JSONTest {
@Test
@SneakyThrows
public void consistencyTest() {
System.setProperty("org.oggio88.javason.value.ObjectValue.preserveKeyOrder", "true");
System.setProperty(ObjectValue.class.getName() + ".preserveKeyOrder", "true");
for (String testFile : testFiles) {
Parser parser = new JSONParser();
Value parsedValue = parser.parse(getTestSource(testFile));
@@ -209,8 +208,8 @@ public class JSONTest {
@Test
@SneakyThrows
public void comparativeTest() {
ObjectMapper om = new ObjectMapper();
for (String testFile : testFiles) {
ObjectMapper om = new ObjectMapper();
JsonNode jsonNode = om.readTree(getTestSource(testFile));
Value value = new JSONParser().parse(getTestSource(testFile));
Assert.assertTrue(compareValueAndJsonNode(value, jsonNode, (v, j) -> {
@@ -223,7 +222,7 @@ public class JSONTest {
@SneakyThrows
public void hexTest() {
String hex = "1F608";
byte[] buffer = new String(hex).getBytes();
byte[] buffer = hex.getBytes();
ByteArrayInputStream bais = new ByteArrayInputStream(buffer);
Method method = JSONParser.class.getDeclaredMethod("parseHex", LookAheadTextInputStream.class);
method.setAccessible(true);

42
src/test/java/org/oggio88/worth/serialization/json/PerformanceTest.java
View File

@@ -4,19 +4,21 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Ignore;
import org.junit.Test;
import org.oggio88.worth.antlr.JSONLexer;
import org.oggio88.worth.antlr.JSONListenerImpl;
import org.oggio88.worth.serialization.binary.JBONDumper;
import org.oggio88.worth.value.ObjectValue;
import org.oggio88.worth.xface.Dumper;
import org.oggio88.worth.xface.Value;
import org.tukaani.xz.XZInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.*;
import java.nio.file.Paths;
class Chronometer {
@@ -54,17 +56,8 @@ class Chronometer {
public class PerformanceTest {
@SneakyThrows
private static byte[] extractTestData() {
ByteArrayOutputStream baous = new ByteArrayOutputStream();
byte[] buffer = new byte[1024 * 1024];
try (InputStream is = new XZInputStream(PerformanceTest.class.getResourceAsStream("/citylots.json.xz"))) {
while (true) {
int read = is.read(buffer);
if (read < 0) break;
baous.write(buffer, 0, read);
}
}
return baous.toByteArray();
private static InputStream extractTestData() {
return new XZInputStream(PerformanceTest.class.getResourceAsStream("/citylots.json.xz"));
}
@Test
@@ -83,9 +76,9 @@ public class PerformanceTest {
final int loops = 100;
Chronometer chr = new Chronometer();
{
ObjectMapper om = new ObjectMapper();
chr.reset();
for (int i = 0; i < loops; i++) {
ObjectMapper om = new ObjectMapper();
JsonNode jsonNode = om.readTree(getClass().getResourceAsStream("/wordpress.json"));
}
jacksonTime = chr.stop(Chronometer.TimeUnit.MILLISECOND);
@@ -102,8 +95,8 @@ public class PerformanceTest {
{
chr.reset();
for (int i = 0; i < loops; i++) {
ANTLRInputStream inputStream = new ANTLRInputStream(
getClass().getResourceAsStream("/wordpress.json"));
CharStream inputStream = CharStreams.fromReader(
new InputStreamReader(getClass().getResourceAsStream("/wordpress.json")));
JSONLexer lexer = new JSONLexer(inputStream);
CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
org.oggio88.worth.antlr.JSONParser parser = new org.oggio88.worth.antlr.JSONParser(commonTokenStream);
@@ -120,25 +113,24 @@ public class PerformanceTest {
@Ignore
@SneakyThrows
public void hugeJSONTest() {
byte[] testData = extractTestData();
double jacksonTime, worthTime, antlrTime;
Chronometer chr = new Chronometer();
{
try(InputStream is = extractTestData()) {
chr.reset();
ObjectMapper om = new ObjectMapper();
JsonNode jsonNode = om.readTree(new ByteArrayInputStream(testData));
om.readTree(is);
jacksonTime = chr.stop(Chronometer.TimeUnit.SECOND);
System.out.printf("Jackson time: %8s sec\n", String.format("%.3f", jacksonTime));
}
{
try(InputStream is = extractTestData()) {
chr.reset();
Value value = new JSONParser().parse(new ByteArrayInputStream(testData));
new JSONParser().parse(is);
worthTime = chr.stop(Chronometer.TimeUnit.SECOND);
System.out.printf("Worth time: %8s sec\n", String.format("%.3f", worthTime));
}
{
try(InputStream is = extractTestData()) {
chr.reset();
ANTLRInputStream inputStream = new ANTLRInputStream(new ByteArrayInputStream(testData));
CharStream inputStream = CharStreams.fromReader(new InputStreamReader(is));
JSONLexer lexer = new JSONLexer(inputStream);
CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
org.oggio88.worth.antlr.JSONParser parser = new org.oggio88.worth.antlr.JSONParser(commonTokenStream);

60
src/test/java/org/oggio88/worth/utils/Leb128Test.java Normal file
View File

@@ -0,0 +1,60 @@
package org.oggio88.worth.utils;
import lombok.SneakyThrows;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
public class Leb128Test {
@Test
public void testLong() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
List<Long> numbers = Arrays.asList(0L, 1L, -3L, 5L, 7L, 8L, 125L, 255L, 10325L, -2000L, 1024L * 1024L * 1024L * 12L);
numbers.forEach(n -> Leb128.encode(baos, n));
byte[] bytes = baos.toByteArray();
Leb128.Leb128Decoder decoder = new Leb128.Leb128Decoder(new ByteArrayInputStream(bytes));
numbers.forEach(n -> Assert.assertEquals((long) n, decoder.decode()));
}
@Test
public void testDouble() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
List<Double> numbers = Arrays.asList(0.0, 1.5, -3.0, 0.5, 2.5, 8.25, -125.0, 255.0, 10325.0, -2000.0, 1024.0 * 1024 * 1024 * 12);
numbers.forEach(n -> Leb128.encode(baos, n));
byte[] bytes = baos.toByteArray();
Leb128.Leb128Decoder decoder = new Leb128.Leb128Decoder(new ByteArrayInputStream(bytes));
numbers.forEach(n -> Assert.assertEquals(n, decoder.decodeDouble(), 0.0));
}
@Test
public void reverseTest() {
long n = 101325;
Assert.assertEquals(n, Leb128.reverse(Leb128.reverse(n)));
}
@Test
@SneakyThrows
public void reverseTestDouble() {
double n = 0.25;
long doubleLong = Double.doubleToLongBits(n);
long reverse = Leb128.reverse(doubleLong);
try(ByteArrayOutputStream os = new ByteArrayOutputStream()) {
Leb128.encode(os, reverse);
byte[] bytes = os.toByteArray();
Assert.assertEquals(3, bytes.length);
}
}
}

29
src/test/java/org/oggio88/worth/utils/ValueWalkerTest.java Normal file
View File

@@ -0,0 +1,29 @@
package org.oggio88.worth.utils;
import lombok.SneakyThrows;
import org.junit.Assert;
import org.junit.Test;
import org.oggio88.worth.serialization.json.JSONParser;
import org.oggio88.worth.xface.Parser;
import org.oggio88.worth.xface.Value;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.util.Optional;
public class ValueWalkerTest {
@Test
@SneakyThrows
public void test() {
Value value;
try(InputStream is = new BufferedInputStream(getClass().getResourceAsStream("/test.json"))) {
Parser parser = JSONParser.newInstance();
value = parser.parse(is);
}
ValueWalker valueWalker = new ValueWalker(value);
Optional<String> text = valueWalker.get("widget").get("image").get("tags").get(1).map(Value::asString);
Assert.assertTrue(text.isPresent());
Assert.assertEquals("Amazon", text.get());
}
}