woggioni/wson
1
0
Fork 0
Code Actions Packages Activity

fixed bug in Leb128 serialization

Browse Source
This commit is contained in:
Walter Oggioni
2019-09-01 23:56:18 +01:00
parent 50089fa6f8
commit 39d6c7ced6
7 changed files with 49 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
antlr/src/test/java/net/woggioni/worth/serialization/json/PerformanceTest.java
View File

@@ -6,6 +6,8 @@ import lombok.SneakyThrows;
import net.woggioni.worth.antlr.JSONLexer;
import net.woggioni.worth.antlr.JSONListenerImpl;
import net.woggioni.worth.serialization.binary.JBONDumper;
import net.woggioni.worth.serialization.binary.JBONParser;
import net.woggioni.worth.value.ObjectValue;
import net.woggioni.worth.xface.Dumper;
import net.woggioni.worth.xface.Parser;
import net.woggioni.worth.xface.Value;
@@ -31,6 +33,11 @@ public class PerformanceTest {
return new XZInputStream(new BufferedInputStream(PerformanceTest.class.getResourceAsStream("/citylots.json.xz")));
}
@SneakyThrows
private static InputStream extractBinaryTestData() {
return new XZInputStream(new BufferedInputStream(PerformanceTest.class.getResourceAsStream("/citylots.jbon.xz")));
}
private static InputStream smallTestData() {
return new BufferedInputStream(PerformanceTest.class.getResourceAsStream("/wordpress.json"));
}
@@ -95,6 +102,9 @@ public class PerformanceTest {
public void hugeJSONTest() {
double jacksonTime, worthTime, antlrTime;
Chronometer chr = new Chronometer();
Value.Configuration cfg = Value.Configuration.builder()
.objectValueImplementation(ObjectValue.Implementation.ArrayList)
.build();
try(InputStream is = extractTestData()) {
chr.reset();
ObjectMapper om = new ObjectMapper();
@@ -104,10 +114,18 @@ public class PerformanceTest {
}
try(InputStream is = extractTestData()) {
chr.reset();
new JSONParser().parse(is);
new JSONParser(cfg).parse(is);
worthTime = chr.stop(Chronometer.TimeUnit.SECOND);
System.out.printf("Worth time: %8s sec\n", String.format("%.3f", worthTime));
}
try(InputStream is = extractBinaryTestData()) {
chr.reset();
new JBONParser(cfg).parse(is);
worthTime = chr.stop(Chronometer.TimeUnit.SECOND);
System.out.printf("Worth time binary: %8s sec\n", String.format("%.3f", worthTime));
}
try(InputStream is = extractTestData()) {
chr.reset();
CharStream inputStream = CharStreams.fromReader(new InputStreamReader(is));

8
src/main/java/net/woggioni/worth/serialization/ValueParser.java
View File

@@ -5,14 +5,18 @@ import net.woggioni.worth.exception.MaxDepthExceededException;
import net.woggioni.worth.exception.NotImplementedException;
import net.woggioni.worth.exception.ParseException;
import net.woggioni.worth.utils.WorthUtils;
import net.woggioni.worth.value.*;
import net.woggioni.worth.value.ArrayValue;
import net.woggioni.worth.value.BooleanValue;
import net.woggioni.worth.value.FloatValue;
import net.woggioni.worth.value.IntegerValue;
import net.woggioni.worth.value.ObjectValue;
import net.woggioni.worth.value.StringValue;
import net.woggioni.worth.xface.Parser;
import net.woggioni.worth.xface.Value;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Array;
import java.nio.charset.Charset;
import java.util.ArrayDeque;
import java.util.HashMap;

11
src/main/java/net/woggioni/worth/utils/Leb128.java
View File

@@ -26,10 +26,10 @@ public class Leb128 {
public static int encode(OutputStream os, long input) {
int bytes_written = 0;
long number = input >= 0 ? (input << 1) : (-(input + 1)) << 1 | 1;
while(number > 127) {
while((number & 127L) != number) {
os.write((int) (number & 127) | 128);
bytes_written++;
number >>= 7;
number >>>= 7;
}
os.write((int) number);
return ++bytes_written;
@@ -67,11 +67,10 @@ public class Leb128 {
if(c < 0) {
throw new IllegalArgumentException("Unexpected end of file");
}
byte b = (byte) c;
res |= ((long)(b & 127)) << (i * 7);
if(b >= 0) break;
res |= ((long)(c & 127)) << (i * 7);
if((byte) c >= 0) break;
}
return (res & 1) != 0 ? - (res >> 1) - 1 : (res >> 1);
return (res & 1) != 0 ? - (res >>> 1) - 1 : (res >>> 1);
}
}
}

22
src/test/java/net/woggioni/worth/serialization/binary/JBONTest.java
View File

@@ -9,7 +9,13 @@ import net.woggioni.worth.xface.Value;
import org.junit.Assert;
import org.junit.Test;
import java.io.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -27,26 +33,26 @@ public class JBONTest {
@SneakyThrows
public void consistencyTest() {
Value.Configuration cfg = Value.Configuration.builder()
.objectValueImplementation(ObjectValue.Implementation.TreeMap).build();
.objectValueImplementation(ObjectValue.Implementation.TreeMap).build();
for (String testFile : testFiles) {
Value parsedValue;
try(InputStream is = getTestSource(testFile)) {
try (InputStream is = getTestSource(testFile)) {
Parser parser = new JSONParser(cfg);
parsedValue = parser.parse(is);
}
byte[] dumpedJBON;
try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
JBONDumper.newInstance().dump(parsedValue, baos);
dumpedJBON = baos.toByteArray();
}
Value reParsedValue;
try(InputStream is = new ByteArrayInputStream(dumpedJBON)) {
try (InputStream is = new ByteArrayInputStream(dumpedJBON)) {
Parser parser = new JBONParser(cfg);
reParsedValue = parser.parse(is);
}
Assert.assertEquals(parsedValue, reParsedValue);
byte[] reDumpedJBON;
try(ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
JBONDumper.newInstance().dump(reParsedValue, baos);
reDumpedJBON = baos.toByteArray();
}
@@ -60,13 +66,13 @@ public class JBONTest {
for (String testFile : testFiles) {
Value originalValue = new JSONParser().parse(getTestSource(testFile));
Path outputFile = Files.createTempFile(Paths.get("/tmp"),"worth", null);
Path outputFile = Files.createTempFile(Paths.get("/tmp"), "worth", null);
try (OutputStream os = new FileOutputStream(outputFile.toFile())) {
JBONDumper jbonDumper = new JBONDumper();
jbonDumper.dump(originalValue, os);
}
Value binarySerializedValue;
try(InputStream is = new FileInputStream(outputFile.toFile())) {
try (InputStream is = new FileInputStream(outputFile.toFile())) {
JBONParser jbonParser = new JBONParser();
binarySerializedValue = jbonParser.parse(is);
}

2
src/test/java/net/woggioni/worth/serialization/json/PerformanceTest.java → src/test/java/net/woggioni/worth/serialization/json/Chronometer.java
View File

@@ -1,6 +1,6 @@
package net.woggioni.worth.serialization.json;
class Chronometer {
public class Chronometer {
public enum TimeUnit {
NANOSECOND(1e-9), MICROSECOND(1e-6), MILLISECOND(1e-3), SECOND(1);

5
src/test/java/net/woggioni/worth/utils/Leb128Test.java
View File

@@ -7,6 +7,7 @@ import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class Leb128Test {
@@ -27,7 +28,9 @@ public class Leb128Test {
@Test
public void testDouble() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
List<Double> numbers = Arrays.asList(0.0, 1.5, -3.0, 0.5, 2.5, 8.25, -125.0, 255.0, 10325.0, -2000.0, 1024.0 * 1024 * 1024 * 12);
List<Double> numbers = Arrays.asList(
0.0, 1.5, -3.0, 0.5, 2.5, 8.25, -125.0, 255.0, 10325.0, -2000.0, 1024.0 * 1024 * 1024 * 12,
-122.42200352825247, 37.80848009696725);
numbers.forEach(n -> Leb128.encode(baos, n));

BIN
src/test/resources/citylots.jbon.xz Normal file
View File

Binary file not shown.