From b62c43e892f2076d519a8ac44d7affa1250d1efb Mon Sep 17 00:00:00 2001 From: josua Date: Fri, 24 Jul 2020 11:26:52 +1000 Subject: [PATCH 1/8] Started working on a lookup table system for BdfNamedList to save memory and space --- src/bdf/classes/BdfClassManager.java | 39 ------- src/bdf/file/BdfFileManager.java | 3 +- src/bdf/types/BdfArray.java | 27 ++++- src/bdf/types/BdfLookupTable.java | 88 +++++++++++++++ src/bdf/types/BdfNamedList.java | 62 ++++++----- src/bdf/types/BdfObject.java | 157 ++++----------------------- src/bdf/types/BdfReader.java | 106 ++++++++++++++++++ src/bdf/types/BdfTypes.java | 23 ++-- src/tests/Tests.java | 56 +--------- 9 files changed, 288 insertions(+), 273 deletions(-) delete mode 100755 src/bdf/classes/BdfClassManager.java create mode 100644 src/bdf/types/BdfLookupTable.java create mode 100644 src/bdf/types/BdfReader.java diff --git a/src/bdf/classes/BdfClassManager.java b/src/bdf/classes/BdfClassManager.java deleted file mode 100755 index 28cff0e..0000000 --- a/src/bdf/classes/BdfClassManager.java +++ /dev/null @@ -1,39 +0,0 @@ -package bdf.classes; - -import bdf.types.BdfObject; - -public class BdfClassManager -{ - protected IBdfClassManager method; - protected BdfObject object = new BdfObject(); - - public BdfClassManager(IBdfClassManager method) - { - // Save some variables for later - this.method = method; - } - - public void setBdf(BdfObject bdf) { - this.object = bdf; - } - - public BdfObject getBdf() { - return this.object; - } - - public void save(BdfObject bdf) { - method.BdfClassSave(bdf); - } - - public void load(BdfObject bdf) { - method.BdfClassLoad(bdf); - } - - public void save() { - this.save(this.object); - } - - public void load() { - this.load(this.object); - } -} diff --git a/src/bdf/file/BdfFileManager.java b/src/bdf/file/BdfFileManager.java index 4c46ca9..f03ff54 100644 --- a/src/bdf/file/BdfFileManager.java +++ b/src/bdf/file/BdfFileManager.java @@ -8,9 +8,10 @@ import java.util.zip.DeflaterOutputStream; import bdf.data.BdfDatabase; import bdf.types.BdfObject; +import bdf.types.BdfReader; import bdf.util.FileHelpers; -public class BdfFileManager extends BdfObject +public class BdfFileManager extends BdfReader { protected String path; private boolean compressed; diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index 5ff46e3..bef4473 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -11,12 +11,16 @@ import bdf.util.DataHelpers; public class BdfArray implements IBdfType, Iterable { protected ArrayList elements = new ArrayList(); + protected BdfLookupTable lookupTable; - public BdfArray() { + BdfArray(BdfLookupTable lookupTable) { + this.lookupTable = lookupTable; } - public BdfArray(IBdfDatabase data) + BdfArray(BdfLookupTable lookupTable, IBdfDatabase data) { + this.lookupTable = lookupTable; + // Create an iterator value to loop over the data int i = 0; @@ -27,7 +31,7 @@ public class BdfArray implements IBdfType, Iterable int size = DataHelpers.getByteBuffer(data.getPointer(i, Integer.BYTES)).getInt(); // Get the object - BdfObject object = new BdfObject(data.getPointer((i+Integer.BYTES), size)); + BdfObject object = new BdfObject(lookupTable, data.getPointer((i+Integer.BYTES), size)); // Add the object to the elements list elements.add(object); @@ -37,6 +41,18 @@ public class BdfArray implements IBdfType, Iterable } } + public BdfObject createObject() { + return new BdfObject(lookupTable); + } + + public BdfNamedList createNamedList() { + return new BdfNamedList(lookupTable); + } + + public BdfArray createArray() { + return new BdfArray(lookupTable); + } + @Override public int serializeSeeker() { @@ -119,9 +135,10 @@ public class BdfArray implements IBdfType, Iterable return this; } - public BdfArray remove(int index) { + public BdfObject remove(int index) { + BdfObject bdf = elements.get(index); elements.remove(index); - return this; + return bdf; } public BdfArray remove(BdfObject bdf) { diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java new file mode 100644 index 0000000..6d12055 --- /dev/null +++ b/src/bdf/types/BdfLookupTable.java @@ -0,0 +1,88 @@ +package bdf.types; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; + +import bdf.data.IBdfDatabase; +import bdf.util.DataHelpers; + +class BdfLookupTable implements IBdfType +{ + private ArrayList keys; + + BdfLookupTable() { + keys = new ArrayList(); + } + + BdfLookupTable(IBdfDatabase database) + { + keys = new ArrayList(); + + for(int i=0;i elements = new ArrayList(); + protected BdfLookupTable lookupTable; - public BdfNamedList() { + BdfNamedList(BdfLookupTable lookupTable) { + this.lookupTable = lookupTable; } - public BdfNamedList(IBdfDatabase data) + BdfNamedList(BdfLookupTable lookupTable, IBdfDatabase data) { + this.lookupTable = lookupTable; + // Create an iterator value to loop over the data int i = 0; @@ -30,15 +34,13 @@ public class BdfNamedList implements IBdfType while(i < data.size()) { // Get the key - int key_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); + int key = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); i += 4; - byte[] key = data.getPointer(i, key_size).getBytes(); // Get the object - i += key_size; int object_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); i += 4; - BdfObject object = new BdfObject(data.getPointer(i, object_size)); + BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size)); // Create a new element and save some data to it Element element = new Element(); @@ -60,14 +62,10 @@ public class BdfNamedList implements IBdfType for(Element o : elements) { - database.setBytes(pos, DataHelpers.serializeInt(o.key.length)); + database.setBytes(pos, DataHelpers.serializeInt(o.key)); pos += 4; - database.setBytes(pos, o.key); - - pos += o.key.length; - int size = o.object.serialize(database.getPointer(pos + 4, database.size() - (pos + 4))); database.setBytes(pos, DataHelpers.serializeInt(size)); @@ -87,7 +85,6 @@ public class BdfNamedList implements IBdfType for(Element o : elements) { size += 8; - size += o.key.length; size += o.object.serializeSeeker(); } @@ -114,7 +111,7 @@ public class BdfNamedList implements IBdfType stream.write(indent.indent.getBytes()); } - stream.write((DataHelpers.serializeString(new String(e.key, StandardCharsets.UTF_8)) + ": ").getBytes()); + stream.write((DataHelpers.serializeString(lookupTable.getName(e.key)) + ": ").getBytes()); e.object.serializeHumanReadable(stream, indent, it + 1); if(elements.size() > i+1) { @@ -143,7 +140,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is this the element key - if(DataHelpers.bytesAreEqual(e.key, key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) { // Set the object object = e.object; @@ -154,7 +151,7 @@ public class BdfNamedList implements IBdfType } // Get a bdf object - BdfObject o = new BdfObject(); + BdfObject o = new BdfObject(lookupTable); // Set the bdf object this.set(key, o); @@ -163,7 +160,7 @@ public class BdfNamedList implements IBdfType return o; } - public BdfNamedList remove(String key) + public BdfObject remove(String key) { // Convert the key to bytes byte[] key_bytes = key.getBytes(); @@ -175,18 +172,13 @@ public class BdfNamedList implements IBdfType Element e = elements.get(i); // Is the specified key the same as the elements key - if(DataHelpers.bytesAreEqual(e.key, key_bytes)) - { - // Delete this element - elements.remove(i); - - // Exit out of the function, prevent NullPointException - return this; + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) { + return elements.remove(i).object; } } // Send back nothing - return this; + return null; } public BdfNamedList remove(BdfObject bdf) @@ -201,6 +193,18 @@ public class BdfNamedList implements IBdfType return this; } + public BdfObject createObject() { + return new BdfObject(lookupTable); + } + + public BdfNamedList createNamedList() { + return new BdfNamedList(lookupTable); + } + + public BdfArray createArray() { + return new BdfArray(lookupTable); + } + public BdfNamedList set(String key, BdfObject object) { // Convert the key to bytes @@ -210,7 +214,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is the key here the same as the specified key - if(DataHelpers.bytesAreEqual(e.key, key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) { // Set the new object e.object = object; @@ -222,7 +226,7 @@ public class BdfNamedList implements IBdfType // Create a new element object Element e = new Element(); - e.key = key_bytes; + e.key = lookupTable.getLocation(key); e.object = object; // Add the new element object to the elements list @@ -241,7 +245,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is the elements key the same as the specified key - if(DataHelpers.bytesAreEqual(e.key, key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) { // Send back true to say the element was found return true; @@ -262,7 +266,7 @@ public class BdfNamedList implements IBdfType { // Get the element Element e = elements.get(i); - keys[i] = new String(e.key, StandardCharsets.UTF_8); + keys[i] = lookupTable.getName(e.key); } // Return the list of keys as strings diff --git a/src/bdf/types/BdfObject.java b/src/bdf/types/BdfObject.java index b7a3d5d..7218516 100644 --- a/src/bdf/types/BdfObject.java +++ b/src/bdf/types/BdfObject.java @@ -13,18 +13,17 @@ public class BdfObject implements IBdfType { protected IBdfDatabase database = null; protected Object object = null; - protected byte type = BdfTypes.EMPTY; + protected byte type = BdfTypes.UNDEFINED; + protected BdfLookupTable lookupTable; - public static BdfObject getNew() { - return new BdfObject(); + BdfObject(BdfLookupTable lookupTable) { + this.lookupTable = lookupTable; } - public BdfObject(byte[] data) { - this(new BdfDatabase(data)); - } - - public BdfObject(IBdfDatabase data) + BdfObject(BdfLookupTable lookupTable, IBdfDatabase data) { + this.lookupTable = lookupTable; + // Is the database length greater than 1 if(data.size() > 1) { @@ -34,8 +33,8 @@ public class BdfObject implements IBdfType // Set the object variable if there is an object specified if(type == BdfTypes.STRING) object = database.getString(); - if(type == BdfTypes.ARRAY) object = new BdfArray(database); - if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(database); + if(type == BdfTypes.ARRAY) object = new BdfArray(lookupTable, database); + if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(lookupTable, database); if(object != null) { database = null; @@ -99,15 +98,6 @@ public class BdfObject implements IBdfType return database.size() + 1; } - public BdfDatabase serialize() - { - BdfDatabase database = new BdfDatabase(serializeSeeker()); - - serialize(database); - - return database; - } - private String calcIndent(BdfIndent indent, int it) { String t = ""; for(int i=0;i<=it;i++) { @@ -116,36 +106,6 @@ public class BdfObject implements IBdfType return t; } - public String serializeHumanReadable(BdfIndent indent) { - return serializeHumanReadable(indent, 0); - } - - public String serializeHumanReadable() { - return serializeHumanReadable(new BdfIndent("", ""), 0); - } - - public String serializeHumanReadable(BdfIndent indent, int it) - { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - - try { - serializeHumanReadable(stream, indent, it); - return stream.toString(); - } - - catch(IOException e) { - return "undefined"; - } - } - - public void serializeHumanReadable(OutputStream stream, BdfIndent indent) throws IOException { - serializeHumanReadable(stream, indent, 0); - } - - public void serializeHumanReadable(OutputStream stream) throws IOException { - serializeHumanReadable(stream, new BdfIndent("", ""), 0); - } - public void serializeHumanReadable(OutputStream stream, BdfIndent indent, int it) throws IOException { String str = null; @@ -280,10 +240,6 @@ public class BdfObject implements IBdfType } } - public BdfObject() { - database = new BdfDatabase(0); - } - public byte getType() { return this.type; } @@ -492,7 +448,7 @@ public class BdfObject implements IBdfType public BdfArray getArray() { if(this.type != BdfTypes.ARRAY) - this.setArray(); + setArray(createArray()); return (BdfArray)object; } @@ -500,7 +456,7 @@ public class BdfObject implements IBdfType public BdfNamedList getNamedList() { if(this.type != BdfTypes.NAMED_LIST) - this.setNamedList(); + setNamedList(createNamedList()); return (BdfNamedList)object; } @@ -577,12 +533,16 @@ public class BdfObject implements IBdfType return this; } - public BdfObject setArray() { - return this.setArray(new BdfArray()); + public BdfObject createObject() { + return new BdfObject(lookupTable); } - public BdfObject setNamedList() { - return this.setNamedList(new BdfNamedList()); + public BdfNamedList createNamedList() { + return new BdfNamedList(lookupTable); + } + + public BdfArray createArray() { + return new BdfArray(lookupTable); } public BdfObject setBooleanArray(boolean[] value) { @@ -654,84 +614,5 @@ public class BdfObject implements IBdfType database = DataHelpers.getDatabase(b); return this; } - - // Primitives - public static BdfObject withInteger(int v) { - return (new BdfObject()).setInteger(v); - } - - public static BdfObject withByte(byte v) { - return (new BdfObject()).setByte(v); - } - - public static BdfObject withBoolean(boolean v) { - return (new BdfObject()).setBoolean(v); - } - - public static BdfObject withFloat(float v) { - return (new BdfObject()).setFloat(v); - } - - public static BdfObject withDouble(double v) { - return (new BdfObject()).setDouble(v); - } - - public static BdfObject withLong(long v) { - return (new BdfObject()).setLong(v); - } - - public static BdfObject withShort(short v) { - return (new BdfObject()).setShort(v); - } - - // Arrays - public static BdfObject withIntegerArray(int[] v) { - return (new BdfObject()).setIntegerArray(v); - } - - public static BdfObject withByteArray(byte[] v) { - return (new BdfObject()).setByteArray(v); - } - - public static BdfObject withBooleanArray(boolean[] v) { - return (new BdfObject()).setBooleanArray(v); - } - - public static BdfObject withFloatArray(float[] v) { - return (new BdfObject()).setFloatArray(v); - } - - public static BdfObject withDoubleArray(double[] v) { - return (new BdfObject()).setDoubleArray(v); - } - - public static BdfObject withLongArray(long[] v) { - return (new BdfObject()).setLongArray(v); - } - - public static BdfObject withShortArray(short[] v) { - return (new BdfObject()).setShortArray(v); - } - - // Objects - public static BdfObject withString(String v) { - return (new BdfObject()).setString(v); - } - - public static BdfObject withArray(BdfArray v) { - return (new BdfObject()).setArray(v); - } - - public static BdfObject withNamedList(BdfNamedList v) { - return (new BdfObject()).setNamedList(v); - } - - public static BdfObject withArray() { - return (new BdfObject()).setArray(new BdfArray()); - } - - public static BdfObject withNamedList() { - return (new BdfObject()).setNamedList(new BdfNamedList()); - } } diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java new file mode 100644 index 0000000..f562555 --- /dev/null +++ b/src/bdf/types/BdfReader.java @@ -0,0 +1,106 @@ +package bdf.types; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import bdf.data.BdfDatabase; +import bdf.data.IBdfDatabase; +import bdf.util.DataHelpers; + +public class BdfReader +{ + protected BdfLookupTable lookupTable; + protected BdfObject bdf; + + public BdfReader() { + lookupTable = new BdfLookupTable(); + bdf = new BdfObject(lookupTable); + } + + public BdfReader(byte[] database) { + this(new BdfDatabase(database)); + } + + public BdfReader(IBdfDatabase database) + { + // Check the version of the BDF file + if(!"BDF3".contentEquals(new String(database.getBytes(0, 4)))) { + lookupTable = new BdfLookupTable(); + bdf = new BdfObject(lookupTable); + return; + } + + // Get the lookup table + int lookupTable_size = DataHelpers.getByteBuffer(database.getPointer(4, 4)).getInt(); + lookupTable = new BdfLookupTable(database.getPointer(8, lookupTable_size)); + + // Get the rest of the data + int upto = lookupTable_size + 8; + int bdf_size = DataHelpers.getByteBuffer(database.getPointer(upto, 4)).getInt(); + bdf = new BdfObject(lookupTable, database.getPointer(upto + 4, bdf_size)); + } + + public BdfDatabase serialize() + { + int bdf_size = bdf.serializeSeeker(); + int lookupTable_size = lookupTable.serializeSeeker(); + int database_size = bdf_size + lookupTable_size + 12; + BdfDatabase database = new BdfDatabase(database_size); + + database.setBytes(0, "BDF3".getBytes()); + database.setBytes(4, DataHelpers.serializeInt(lookupTable_size)); + database.setBytes(8 + lookupTable_size, DataHelpers.serializeInt(bdf_size)); + + lookupTable.serialize(database.getPointer(8)); + bdf.serialize(database.getPointer(12 + lookupTable_size)); + + return database; + } + + public BdfObject getBDF() { + return bdf; + } + + public String serializeHumanReadable(BdfIndent indent) { + return serializeHumanReadable(indent, 0); + } + + public String serializeHumanReadable() { + return serializeHumanReadable(new BdfIndent("", ""), 0); + } + + public String serializeHumanReadable(BdfIndent indent, int it) + { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + try { + bdf.serializeHumanReadable(stream, indent, it); + return stream.toString(); + } + + catch(IOException e) { + return "undefined"; + } + } + + public void serializeHumanReadable(OutputStream stream, BdfIndent indent) throws IOException { + bdf.serializeHumanReadable(stream, indent, 0); + } + + public void serializeHumanReadable(OutputStream stream) throws IOException { + bdf.serializeHumanReadable(stream, new BdfIndent("", ""), 0); + } + + public BdfObject createObject() { + return new BdfObject(lookupTable); + } + + public BdfNamedList createNamedList() { + return new BdfNamedList(lookupTable); + } + + public BdfArray createArray() { + return new BdfArray(lookupTable); + } +} diff --git a/src/bdf/types/BdfTypes.java b/src/bdf/types/BdfTypes.java index adc5a2b..1a1d786 100644 --- a/src/bdf/types/BdfTypes.java +++ b/src/bdf/types/BdfTypes.java @@ -2,18 +2,19 @@ package bdf.types; public class BdfTypes { - public static final byte BOOLEAN = 0; - public static final byte INTEGER = 1; - public static final byte LONG = 2; - public static final byte SHORT = 3; - public static final byte BYTE = 4; - public static final byte DOUBLE = 5; - public static final byte FLOAT = 6; + public static final byte UNDEFINED = 0; - public static final byte STRING = 7; - public static final byte ARRAY = 8; - public static final byte NAMED_LIST = 9; - public static final byte EMPTY = 10; + public static final byte BOOLEAN = 1; + public static final byte INTEGER = 2; + public static final byte LONG = 3; + public static final byte SHORT = 4; + public static final byte BYTE = 5; + public static final byte DOUBLE = 6; + public static final byte FLOAT = 7; + + public static final byte STRING = 8; + public static final byte ARRAY = 9; + public static final byte NAMED_LIST = 10; public static final byte ARRAY_BOOLEAN = 11; public static final byte ARRAY_INTEGER = 12; diff --git a/src/tests/Tests.java b/src/tests/Tests.java index ca7d29b..63c2be6 100755 --- a/src/tests/Tests.java +++ b/src/tests/Tests.java @@ -1,67 +1,23 @@ package tests; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import bdf.data.BdfDatabase; -import bdf.file.BdfFileManager; -import bdf.types.BdfArray; import bdf.types.BdfIndent; import bdf.types.BdfNamedList; import bdf.types.BdfObject; +import bdf.types.BdfReader; public class Tests { public static void main(String[] args) throws InterruptedException, IOException { - /* - BdfObject bdf = new BdfObject(); + BdfReader reader = new BdfReader(); + BdfObject bdf = reader.getBDF(); BdfNamedList nl = bdf.getNamedList(); + nl.set("hello", nl.createObject().setInteger(69)); + nl.set("world", nl.createObject().setInteger(420)); - byte[] bytes = new byte[1024*1024*1024]; - for(int i=0;i Date: Fri, 24 Jul 2020 17:54:56 +1000 Subject: [PATCH 2/8] Added emoji support --- src/bdf/types/BdfLookupTable.java | 26 ++++++++++++-------------- src/bdf/types/BdfNamedList.java | 16 ++++++++-------- src/bdf/types/BdfObject.java | 8 ++++---- src/bdf/types/BdfReader.java | 4 ++-- src/bdf/util/DataHelpers.java | 11 +++-------- src/tests/Tests.java | 26 ++++++++++++++++++++++++-- 6 files changed, 53 insertions(+), 38 deletions(-) diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index 6d12055..68c34bd 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -10,34 +10,32 @@ import bdf.util.DataHelpers; class BdfLookupTable implements IBdfType { - private ArrayList keys; + private ArrayList keys; BdfLookupTable() { - keys = new ArrayList(); + keys = new ArrayList(); } BdfLookupTable(IBdfDatabase database) { - keys = new ArrayList(); + keys = new ArrayList(); for(int i=0;i i+1) { @@ -140,7 +140,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is this the element key - if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key), key_bytes)) { // Set the object object = e.object; @@ -172,7 +172,7 @@ public class BdfNamedList implements IBdfType Element e = elements.get(i); // Is the specified key the same as the elements key - if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) { + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key), key_bytes)) { return elements.remove(i).object; } } @@ -214,7 +214,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is the key here the same as the specified key - if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key), key_bytes)) { // Set the new object e.object = object; @@ -226,7 +226,7 @@ public class BdfNamedList implements IBdfType // Create a new element object Element e = new Element(); - e.key = lookupTable.getLocation(key); + e.key = lookupTable.getLocation(key_bytes); e.object = object; // Add the new element object to the elements list @@ -245,7 +245,7 @@ public class BdfNamedList implements IBdfType for(Element e : elements) { // Is the elements key the same as the specified key - if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) + if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key), key_bytes)) { // Send back true to say the element was found return true; @@ -266,7 +266,7 @@ public class BdfNamedList implements IBdfType { // Get the element Element e = elements.get(i); - keys[i] = lookupTable.getName(e.key); + keys[i] = new String(lookupTable.getName(e.key)); } // Return the list of keys as strings diff --git a/src/bdf/types/BdfObject.java b/src/bdf/types/BdfObject.java index 7218516..829ade4 100644 --- a/src/bdf/types/BdfObject.java +++ b/src/bdf/types/BdfObject.java @@ -67,9 +67,9 @@ public class BdfObject implements IBdfType break; case BdfTypes.STRING: - String str = (String)object; - size = str.length() + 1; - db.setBytes(0, str.getBytes()); + byte[] str = ((String)object).getBytes(); + size = str.length + 1; + db.setBytes(0, str); break; default: @@ -91,7 +91,7 @@ public class BdfObject implements IBdfType { case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker() + 1; case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker() + 1; - case BdfTypes.STRING: return ((String)object).length() + 1; + case BdfTypes.STRING: return ((String)object).getBytes().length + 1; } // Anything else diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java index f562555..3b5ba2b 100644 --- a/src/bdf/types/BdfReader.java +++ b/src/bdf/types/BdfReader.java @@ -52,8 +52,8 @@ public class BdfReader database.setBytes(4, DataHelpers.serializeInt(lookupTable_size)); database.setBytes(8 + lookupTable_size, DataHelpers.serializeInt(bdf_size)); - lookupTable.serialize(database.getPointer(8)); - bdf.serialize(database.getPointer(12 + lookupTable_size)); + lookupTable.serialize(database.getPointer(8, lookupTable_size)); + bdf.serialize(database.getPointer(12 + lookupTable_size, database_size)); return database; } diff --git a/src/bdf/util/DataHelpers.java b/src/bdf/util/DataHelpers.java index e57c00e..7e89b5d 100644 --- a/src/bdf/util/DataHelpers.java +++ b/src/bdf/util/DataHelpers.java @@ -38,17 +38,16 @@ public class DataHelpers return true; } - public static String replaceInString(String string, byte find, String replace) + public static String replaceInString(String string, char find, String replace) { // Convert the string to bytes - byte[] string_b = string.getBytes(); String string_modified = new String(); // Loop over the string - for(int i=0;i Date: Mon, 27 Jul 2020 11:11:01 +1000 Subject: [PATCH 3/8] Got a functional version of BDF 3 --- src/bdf/file/BdfFileManager.java | 4 +- src/bdf/types/BdfArray.java | 27 +++++------ src/bdf/types/BdfLookupTable.java | 52 ++++++++++++++++++--- src/bdf/types/BdfNamedList.java | 43 ++++++----------- src/bdf/types/BdfObject.java | 75 +++++++++++++++--------------- src/bdf/types/BdfReader.java | 76 +++++++++++++++---------------- src/bdf/types/IBdfType.java | 5 +- src/bdf/util/FileHelpers.java | 3 +- src/tests/Tests.java | 44 ++++++++++++------ 9 files changed, 180 insertions(+), 149 deletions(-) diff --git a/src/bdf/file/BdfFileManager.java b/src/bdf/file/BdfFileManager.java index f03ff54..0a78daa 100644 --- a/src/bdf/file/BdfFileManager.java +++ b/src/bdf/file/BdfFileManager.java @@ -1,10 +1,12 @@ package bdf.file; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPOutputStream; import bdf.data.BdfDatabase; import bdf.types.BdfObject; @@ -65,7 +67,7 @@ public class BdfFileManager extends BdfReader OutputStream out = new FileOutputStream(path); if(compressed) { - out = new DeflaterOutputStream(out); + out = new GZIPOutputStream(out); } // Write the database to the file diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index bef4473..eb857d9 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -41,25 +41,13 @@ public class BdfArray implements IBdfType, Iterable } } - public BdfObject createObject() { - return new BdfObject(lookupTable); - } - - public BdfNamedList createNamedList() { - return new BdfNamedList(lookupTable); - } - - public BdfArray createArray() { - return new BdfArray(lookupTable); - } - @Override - public int serializeSeeker() + public int serializeSeeker(int[] locations) { int size = 0; for(BdfObject o : elements) { - size += o.serializeSeeker(); + size += o.serializeSeeker(locations); size += 4; } @@ -67,13 +55,13 @@ public class BdfArray implements IBdfType, Iterable } @Override - public int serialize(IBdfDatabase database) + public int serialize(IBdfDatabase database, int[] locations) { int pos = 0; for(BdfObject o : elements) { - int size = o.serialize(database.getPointer(pos + 4)); + int size = o.serialize(database.getPointer(pos + 4), locations); database.setBytes(pos, DataHelpers.serializeInt(size)); pos += size; @@ -190,5 +178,12 @@ public class BdfArray implements IBdfType, Iterable } }; } + + @Override + public void getLocationUses(int[] locations) { + for(BdfObject element : elements) { + element.getLocationUses(locations); + } + } } diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index 68c34bd..ed60d51 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -11,14 +11,18 @@ import bdf.util.DataHelpers; class BdfLookupTable implements IBdfType { private ArrayList keys; + private BdfReader reader; - BdfLookupTable() { - keys = new ArrayList(); + BdfLookupTable(BdfReader reader) + { + this.keys = new ArrayList(); + this.reader = reader; } - BdfLookupTable(IBdfDatabase database) + BdfLookupTable(BdfReader reader, IBdfDatabase database) { - keys = new ArrayList(); + this.keys = new ArrayList(); + this.reader = reader; for(int i=0;i 0) { + locations[i] = next; + next += 1; + } else { + locations[i] = -1; + } + } + + return locations; + } @Override public void serializeHumanReadable(OutputStream stream, BdfIndent indent, int it) { } + + @Override + public void getLocationUses(int[] locations) { + } } diff --git a/src/bdf/types/BdfNamedList.java b/src/bdf/types/BdfNamedList.java index c8c5aea..3189023 100644 --- a/src/bdf/types/BdfNamedList.java +++ b/src/bdf/types/BdfNamedList.java @@ -56,17 +56,17 @@ public class BdfNamedList implements IBdfType } @Override - public int serialize(IBdfDatabase database) + public int serialize(IBdfDatabase database, int[] locations) { int pos = 0; for(Element o : elements) { - database.setBytes(pos, DataHelpers.serializeInt(o.key)); + database.setBytes(pos, DataHelpers.serializeInt(locations[o.key])); pos += 4; - int size = o.object.serialize(database.getPointer(pos + 4)); + int size = o.object.serialize(database.getPointer(pos + 4), locations); database.setBytes(pos, DataHelpers.serializeInt(size)); @@ -78,14 +78,14 @@ public class BdfNamedList implements IBdfType } @Override - public int serializeSeeker() + public int serializeSeeker(int[] locations) { int size = 0; for(Element o : elements) { size += 8; - size += o.object.serializeSeeker(); + size += o.object.serializeSeeker(locations); } return size; @@ -181,30 +181,6 @@ public class BdfNamedList implements IBdfType return null; } - public BdfNamedList remove(BdfObject bdf) - { - for(int i=0;i 1) - { - // Get the type and database values - type = data.getByte(0); - database = data.getPointer(1, data.size() - 1); - - // Set the object variable if there is an object specified - if(type == BdfTypes.STRING) object = database.getString(); - if(type == BdfTypes.ARRAY) object = new BdfArray(lookupTable, database); - if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(lookupTable, database); - - if(object != null) { - database = null; - } - } + // Get the type and database values + type = data.getByte(0); + database = data.getPointer(1, data.size() - 1); - else - { - // Create a new database - database = new BdfDatabase(0); + // Set the object variable if there is an object specified + if(type == BdfTypes.STRING) object = database.getString(); + if(type == BdfTypes.ARRAY) object = new BdfArray(lookupTable, database); + if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(lookupTable, database); + + if(object != null) { + database = null; } } @Override - public int serialize(IBdfDatabase database) + public void getLocationUses(int[] locations) + { + if(type == BdfTypes.NAMED_LIST || type == BdfTypes.ARRAY) { + ((IBdfType)object).getLocationUses(locations); + } + } + + @Override + public int serialize(IBdfDatabase database, int[] locations) { int size; @@ -59,11 +58,11 @@ public class BdfObject implements IBdfType switch(type) { case BdfTypes.ARRAY: - size = ((BdfArray)object).serialize(db) + 1; + size = ((BdfArray)object).serialize(db, locations) + 1; break; case BdfTypes.NAMED_LIST: - size = ((BdfNamedList)object).serialize(db) + 1; + size = ((BdfNamedList)object).serialize(db, locations) + 1; break; case BdfTypes.STRING: @@ -84,13 +83,13 @@ public class BdfObject implements IBdfType } @Override - public int serializeSeeker() + public int serializeSeeker(int[] locations) { // Objects switch(type) { - case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker() + 1; - case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker() + 1; + case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker(locations) + 1; + case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker(locations) + 1; case BdfTypes.STRING: return ((String)object).getBytes().length + 1; } @@ -158,7 +157,7 @@ public class BdfObject implements IBdfType int[] array = this.getIntegerArray(); for(int i=0;i Date: Fri, 14 Aug 2020 12:25:04 +1000 Subject: [PATCH 4/8] Trying to compress the format as much as possible --- src/bdf/types/BdfArray.java | 16 ++---- src/bdf/types/BdfNamedList.java | 72 ++++++++++++++------------- src/bdf/types/BdfObject.java | 88 +++++++++++++++++++++++++++------ src/bdf/types/BdfReader.java | 9 ++-- 4 files changed, 122 insertions(+), 63 deletions(-) diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index eb857d9..aa9b645 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -28,16 +28,16 @@ public class BdfArray implements IBdfType, Iterable while(i < data.size()) { // Get the size of the object - int size = DataHelpers.getByteBuffer(data.getPointer(i, Integer.BYTES)).getInt(); + int size = BdfObject.getSize(data.getPointer(i)); // Get the object - BdfObject object = new BdfObject(lookupTable, data.getPointer((i+Integer.BYTES), size)); + BdfObject object = new BdfObject(lookupTable, data.getPointer(i, size)); // Add the object to the elements list elements.add(object); // Increase the iterator by the amount of bytes - i += Integer.BYTES+size; + i += size; } } @@ -48,7 +48,6 @@ public class BdfArray implements IBdfType, Iterable for(BdfObject o : elements) { size += o.serializeSeeker(locations); - size += 4; } return size; @@ -59,13 +58,8 @@ public class BdfArray implements IBdfType, Iterable { int pos = 0; - for(BdfObject o : elements) - { - int size = o.serialize(database.getPointer(pos + 4), locations); - database.setBytes(pos, DataHelpers.serializeInt(size)); - - pos += size; - pos += 4; + for(BdfObject o : elements) { + pos += o.serialize(database.getPointer(pos), locations); } return pos; diff --git a/src/bdf/types/BdfNamedList.java b/src/bdf/types/BdfNamedList.java index 3189023..ab9274e 100644 --- a/src/bdf/types/BdfNamedList.java +++ b/src/bdf/types/BdfNamedList.java @@ -38,8 +38,7 @@ public class BdfNamedList implements IBdfType i += 4; // Get the object - int object_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); - i += 4; + int object_size = BdfObject.getSize(data.getPointer(i)); BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size)); // Create a new element and save some data to it @@ -64,14 +63,9 @@ public class BdfNamedList implements IBdfType { database.setBytes(pos, DataHelpers.serializeInt(locations[o.key])); - pos += 4; - int size = o.object.serialize(database.getPointer(pos + 4), locations); - database.setBytes(pos, DataHelpers.serializeInt(size)); - - pos += 4; - pos += size; + pos += size + 4; } return pos; @@ -84,7 +78,7 @@ public class BdfNamedList implements IBdfType for(Element o : elements) { - size += 8; + size += 4; size += o.object.serializeSeeker(locations); } @@ -128,19 +122,16 @@ public class BdfNamedList implements IBdfType stream.write('}'); } - public BdfObject get(String key) + public BdfObject get(int key) { // Get the object to send back BdfObject object = null; - // Convert the key to bytes - byte[] key_bytes = key.getBytes(); - // Loop over the elements for(Element e : elements) { // Is this the element key - if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key), key_bytes)) + if(e.key == key) { // Set the object object = e.object; @@ -160,11 +151,8 @@ public class BdfNamedList implements IBdfType return o; } - public BdfObject remove(String key) + public BdfObject remove(int key) { - // Convert the key to bytes - byte[] key_bytes = key.getBytes(); - // Loop over the elements for(int i=0;i> 4); + } // Set the object variable if there is an object specified if(type == BdfTypes.STRING) object = database.getString(); @@ -39,6 +46,49 @@ public class BdfObject implements IBdfType } } + private boolean shouldStoreSize(byte b) { + return b > 7; + } + + static int getSize(IBdfDatabase db) + { + byte type = db.getByte(0); + int size = getSize(type); + + if(size != -1) { + return size; + } + + return DataHelpers.getByteBuffer(db.getPointer(1, 4)).getInt() + 5; + } + + static int getSize(byte type) + { + type &= 0x0f; + + switch(type) + { + case BdfTypes.BOOLEAN: + return 2; + case BdfTypes.BYTE: + return 2; + case BdfTypes.DOUBLE: + return 9; + case BdfTypes.FLOAT: + return 5; + case BdfTypes.INTEGER: + return 5; + case BdfTypes.LONG: + return 9; + case BdfTypes.SHORT: + return 3; + case BdfTypes.UNDEFINED: + return 1; + default: + return -1; + } + } + @Override public void getLocationUses(int[] locations) { @@ -52,33 +102,38 @@ public class BdfObject implements IBdfType { int size; - IBdfDatabase db = database.getPointer(1); + boolean storeSize = shouldStoreSize(type); // Objects switch(type) { case BdfTypes.ARRAY: - size = ((BdfArray)object).serialize(db, locations) + 1; + size = ((BdfArray)object).serialize(database.getPointer(5), locations) + 5; break; case BdfTypes.NAMED_LIST: - size = ((BdfNamedList)object).serialize(db, locations) + 1; + size = ((BdfNamedList)object).serialize(database.getPointer(5), locations) + 5; break; case BdfTypes.STRING: byte[] str = ((String)object).getBytes(); - size = str.length + 1; - db.setBytes(0, str); + size = str.length + 5; + database.setBytes(5, str); break; default: - size = this.database.size() + 1; - db.setBytes(0, this.database.getBytes()); + int o = storeSize ? 5 : 1; + size = this.database.size() + o; + database.setBytes(o, this.database.getBytes()); break; } database.setByte(0, type); + if(storeSize) { + database.setBytes(1, DataHelpers.serializeInt(size - 5)); + } + return size; } @@ -88,13 +143,18 @@ public class BdfObject implements IBdfType // Objects switch(type) { - case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker(locations) + 1; - case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker(locations) + 1; - case BdfTypes.STRING: return ((String)object).getBytes().length + 1; + case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker(locations) + 5; + case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker(locations) + 5; + case BdfTypes.STRING: return ((String)object).getBytes().length + 5; } - // Anything else - return database.size() + 1; + int size = getSize(type); + + if(size != -1) { + return size; + } + + return database.size() + 5; } private String calcIndent(BdfIndent indent, int it) { diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java index 7c12bfd..606f8ee 100644 --- a/src/bdf/types/BdfReader.java +++ b/src/bdf/types/BdfReader.java @@ -39,8 +39,8 @@ public class BdfReader // Get the rest of the data int upto = lookupTable_size + 4; - int bdf_size = DataHelpers.getByteBuffer(database.getPointer(upto, 4)).getInt(); - bdf = new BdfObject(lookupTable, database.getPointer(upto + 4, bdf_size)); + int bdf_size = BdfObject.getSize(database.getPointer(upto)); + bdf = new BdfObject(lookupTable, database.getPointer(upto, bdf_size)); } public BdfDatabase serialize() @@ -49,14 +49,13 @@ public class BdfReader int bdf_size = bdf.serializeSeeker(locations); int lookupTable_size = lookupTable.serializeSeeker(locations); - int database_size = bdf_size + lookupTable_size + 8; + int database_size = bdf_size + lookupTable_size + 4; BdfDatabase database = new BdfDatabase(database_size); database.setBytes(0, DataHelpers.serializeInt(lookupTable_size)); - database.setBytes(4 + lookupTable_size, DataHelpers.serializeInt(bdf_size)); lookupTable.serialize(database.getPointer(4, lookupTable_size), locations); - bdf.serialize(database.getPointer(8 + lookupTable_size, database_size), locations); + bdf.serialize(database.getPointer(4 + lookupTable_size, database_size), locations); return database; } From 89426bff9832d740579e09a673ea4ada286a7e70 Mon Sep 17 00:00:00 2001 From: josua Date: Mon, 17 Aug 2020 22:54:22 +1000 Subject: [PATCH 5/8] Added human readable de-serialization, modified the human readable layout slightly to allow for empty primitive arrays, added error handling for human readable data, started working on README --- README.md | 257 +++++------- data.hbdf | 22 + src/bdf/data/BdfDatabase.java | 83 ++-- src/bdf/data/BdfDatabasePointer.java | 160 ++++++++ src/bdf/data/BdfStringPointer.java | 233 +++++++++++ src/bdf/data/IBdfDatabase.java | 13 +- src/bdf/types/BdfArray.java | 55 ++- src/bdf/types/BdfLookupTable.java | 98 ++++- src/bdf/types/BdfNamedList.java | 148 ++++++- src/bdf/types/BdfObject.java | 576 +++++++++++++++++++++++++-- src/bdf/types/BdfReader.java | 103 ++++- src/bdf/types/IBdfType.java | 2 +- src/bdf/util/BdfError.java | 102 +++++ src/bdf/util/DataHelpers.java | 111 +++++- src/tests/Tests.java | 36 +- 15 files changed, 1672 insertions(+), 327 deletions(-) create mode 100644 data.hbdf create mode 100644 src/bdf/data/BdfDatabasePointer.java create mode 100644 src/bdf/data/BdfStringPointer.java create mode 100644 src/bdf/util/BdfError.java diff --git a/README.md b/README.md index 42a5636..ad08ed2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Binary Data Format (or BDF) is designed to store data in a tree-like binary structure, like Notch's NBT format, but also open source and free like JSON. The format is -fast and allows multiple data types, it uses 32-bit integers, so BDF files can +fast and allows multiple data types. It uses 32-bit integers, so BDF files can be fast and work well on 32-bit systems, but have a maximum size of 2 GB. BDF allows human readable serialization to see what is going on for debugging purposes, but it currently can't parse the human readable serialized string to an object. @@ -28,6 +28,7 @@ purposes, but it currently can't parse the human readable serialized string to a ### Data types +- Undefined - Boolean - Integer - Long @@ -35,20 +36,33 @@ purposes, but it currently can't parse the human readable serialized string to a - Byte - Double - Float +- Boolean Array +- Integer Array +- Long Array +- Short Array +- Byte Array +- Double Array +- Float Array - String - Array - Named List -- Empty ### Creating an object -You will need to create a new object to store any data, use a `BdfObject` instance. -You can input serialized data into `BdfObject` via a `BdfDatabase`. +You will need to generate a BdfObject to serialize anything, +this can be done by first generating a BdfReader, or generating +a new object via an existing BdfObject. ```java -// New BDF object -BdfObject bdf = new BdfObject(); +// Create a reader object +BdfReader reader = new BdfReader(); + +// Get the BdfObject instance +BdfObject bdf = reader.getObject(); + +// Generate another BdfObject instance +BdfObject bdf_new = bdf.newObject(); // Get an integer int v = bdf.getInteger(); @@ -56,6 +70,9 @@ int v = bdf.getInteger(); // Set an integer bdf.setInteger(5); +// Set a "smart" integer +bdf.setSmartInteger(53); + // Get the type of variable of the object int type = bdf.getType(); @@ -66,65 +83,58 @@ if(type == BdfTypes.INTEGER) } // Serialize the BDF object -byte[] data = bdf.serialize().getBytes(); - - +IBdfDatabase data = bdf.serialize(); // Load another BDF object with the serialized bytes BdfObject bdf2 = new BdfObject(new BdfDatabase(data)); ``` -A `BdfFileManager` -instance can be used in the same way as a `BdfObject`, but it also needs a String parameter -for the path of the file. The file can be written with `BdfFileManager.saveDatabase()`. -A `BdfFileManager` is an instance of `BdfObject`, a `BdfFileManager` can be casted to -a `BdfObject`. +A file manager instance can be used in the same way as a reader object, +but it also needs a String parameter for the path of the file. The file +manager instance also has the capacity to use compression (by default this +uses the GZIP compression algorithm). ```java -// Open a file -BdfFileManager bdf = new BdfFileManager("file.bdf"); +// Open a file with compression enabled +BdfFileManager reader = new BdfFileManager("file.bdf", true); // Save the database -bdf.saveDatabase(); +reader.saveDatabase(); -// The file can be casted to a BdfObject -BdfObject bdf2 = (BdfObject) bdf; +// The file can be casted to a BdfReader +BdfReader reader2 = (BdfReader) reader; -// Bdf -System.out.println(bdf instanceof BdfObject); // true +// Can be used just as any reader instance +BdfObject bdf = reader.getObject(); ``` ### Arrays -Arrays can be used to store lists of information, they hold `BdfObject`. -The array is called with `new BdfArray()`. It can hold information, get -the size of the array with `BdfArray.size()`, remove elements with -`BdfArray.remove(index)`, set indexes with `BdfArray.set(index, BdfObject)`, -and add elements with `BdfArray.add(BdfObject)`. Arrays also -have support for Iterators and are an instance of `Iterable`. +Arrays can be used to store lists of information, they hold instances of +BdfObject. Arrays have support for Iterators and are an instance of Iterable. ```java -// New BDF Object -BdfObject bdf = new BdfObject(); +BdfReader reader = new BdfReader(); +BdfObject bdf = reader.getObject(); -// New BDF Array -BdfArray array = new BdfArray(); +// Can be created from a bdf object +BdfArray array = bdf.newArray(); -// Size +// Get the length of an array int size = array.size(); -// Remove -array.remove(3); // Could be any number +// Remove any index from an array +array.remove(3); -// Set - Could be any number with any object -array.set(4, BdfObject.withString("A String")); +// Set an object to an index of an array +array.set(4, bdf.newObject().setString("A String")); -// Add - Could be any object -array.add(BdfObject.withByte(53)); +// Add an object to an array +array.add(bdf.newObject().setByte(53)); // Set the array to the bdf object bdf.setArray(array); @@ -139,23 +149,27 @@ for(BdfObject o : array) ### Named lists -Named lists can be used to store data under strings, +Named lists can be used to store data under ids/strings to be used like variables in a program. A named list -can be created with `new BdfNamedList()` and it -has the ability to set with `BdfNamedList.set(String, BdfObject)`, -remove with `BdfNamedList.remove(String)`, and check -for a key with `BdfNamedList.contains(String)`. It also has -features to get all the keys with `BdfNamedList.getKeys()`. -Named lists also have Iterator support and are an instance of -`Iterable`. +can be created similar to an array. ```java -// New bdf named list -BdfNamedList list = new BdfNamedList(); +BdfReader reader = new BdfReader(); +BdfObject bdf = new BdfObject(); -// Set an element with a value -list.set("key1", BdfObject.withInteger(5)); +// New named list +BdfNamedList nl = bdf.newNamedList(); + +// Set an element to the named list +nl.set("key1", bdf.newObject().setInteger(5)); + +// Use ids instead of strings for optimisation +// if set/get is being called multiple times +// on the same key. + +int key2 = nl.getKeyLocation("key2"); +nl.set(key2, bdf.newObject().setFloat(42.0F)); // Get an elements value int v = list.get("key1").getInteger(); @@ -164,125 +178,45 @@ int v = list.get("key1").getInteger(); boolean has_key = list.contains("key1"); // Get the lists keys -String[] keys = list.getKeys(); +int[] keys = list.getKeys(); // Iterate over the lists keys -for(String key : keys) +for(int key : keys) { - + // Get the keys name + String key_name = nl.getKeyName(key); } ``` -### Saving classes +### Further optimisations -Classes can be saved with `BdfClassManager` and by -implementing the `IBdfClassManager` interface, -adding 2 functions `BdfClassLoad` and `BdfClassSave`. -`BdfClassLoad` is for checking and loading data from -bdf into the classes variables, while `BdfClassSave` -is for packing pre-existing variables into bdf format. -A BdfClassManager can be used to pass the `IBdfClassManager` -interface into. - -A class with `IBdfClassManager` to save the data -could look like this: - -```java - -class HelloWorld implements IBdfClassManager -{ - int iterator = 0; - - @Override - public void BdfClassLoad(BdfObject bdf) - { - // Load scripts here - - // Get the named list - BdfNamedList nl = bdf.getNamedList(); - - // Set the iterator stored in bdf - int iterator = nl.get("iterator").getInteger(); - } - - @Override - public void BdfClassSave(BdfObject bdf) - { - // Save scripts here - - // Create a named list - BdfNamedList nl = new BdfNamedList(); - - // Set the iterator to the named list - nl.set("iterator", BdfObject.withInteger(iterator)); - - // Store the named list - bdf.setNamedList(nl); - } - - public void hello() - { - // Increase the iterator by 1 - iterator++; - - // Say "Hello, World! Script executed times!" - System.out.println("Hello, World! Script executed "+iterator+" times!"); - } - -} - -``` - -A script to manage this could look something like this: - -```java - -/* - Get a new BdfObject instance, it could be existing, - or from another file, BdfArray, or BdfNamedList instance. -*/ -BdfObject bdf = new BdfObject(); - -// Create the HelloWorld class -HelloWorld hello = new HelloWorld(); - -// Get a new BdfClassManager instance to deal with BDF data -BdfClassManager manager = new BdfClassManager(hello); - -// Give the manager an existing BdfObject instance -manager.setBdf(bdf); - -// Load the classes bdf data -manager.load(); - -// Call the hello world function -hello.hello(); - -// Save the classes bdf data -manager.save(); - -``` ### Implementation details -All integer data types used are signed and Big Endian. +All integer data types are in the Big Endian layout. -**Type (1 byte)** +**Flags (1 unsigned byte)** +This holds 3 values: +- Type (0-17) +- Size type (0-2) +- Parent payload (0-2) + +**Type** ``` -0: BOOLEAN (1 byte, 0x00 or 0x01) -1: INTEGER (4 bytes) -2: LONG (8 bytes) -3: SHORT (2 bytes) -4: BYTE (1 byte) -5: DOUBLE (8 bytes) -6: FLOAT (4 bytes) +0: UNDEFINED (0 bytes) -7: STRING -8: ARRAY -9: NAMED_LIST +1: BOOLEAN (1 byte, 0x00 or 0x01) +2: INTEGER (4 bytes) +3: LONG (8 bytes) +4: SHORT (2 bytes) +5: BYTE (1 byte) +6: DOUBLE (8 bytes) +7: FLOAT (4 bytes) -10: EMPTY (0 bytes) +8: STRING +9: ARRAY +10: NAMED_LIST 11: ARRAY_BOOLEAN 12: ARRAY_INTEGER @@ -291,18 +225,23 @@ All integer data types used are signed and Big Endian. 15: ARRAY_BYTE 16: ARRAY_DOUBLE 17: ARRAY_FLOAT + ``` +**Size Type** +This value holds info for how big the size of +the size of the payload is, in bytes. The purpose +of this is to reduce the size as much as possible +by throwing out unneccicary zeros. + **Object** -- Type (signed byte, 1 byte) +- Flags (unsigned byte, 1 byte) +- Size (variable length) - Payload (Any type, variable length) **NamedList** -- Key size (signed int, 4 bytes) -- Key (variable length) -- Payload size (signed int, 4 bytes) +- Key ID (variable length) - Payload (Object, variable length) **Array** -- Payload size (signed int, 4 bytes) - Payload (Object, variable length) diff --git a/data.hbdf b/data.hbdf new file mode 100644 index 0000000..bbdc4fb --- /dev/null +++ b/data.hbdf @@ -0,0 +1,22 @@ +{ + "name": "Josua", + "age": 17I, + "anArray": [ + "hi =)", + 69B + ], + "anIntArray": int( + 432I, + 234I, + 69I, + 2I, + 423I + ), + "array2": [ + bool( + false, true , + true , true , + true , false, + ) + ] +} diff --git a/src/bdf/data/BdfDatabase.java b/src/bdf/data/BdfDatabase.java index 650dd02..7648f7a 100644 --- a/src/bdf/data/BdfDatabase.java +++ b/src/bdf/data/BdfDatabase.java @@ -5,16 +5,12 @@ import java.io.OutputStream; public class BdfDatabase implements IBdfDatabase { - private static final int STREAM_CHUNK_SIZE = 1024*1024; + static final int STREAM_CHUNK_SIZE = 1024*1024; - private byte[] database; - private int location; - private int size; + byte[] database; public BdfDatabase(byte[] bytes) { database = bytes; - size = bytes.length; - location = 0; } public BdfDatabase(String str) { @@ -23,28 +19,28 @@ public class BdfDatabase implements IBdfDatabase public BdfDatabase(int size) { this.database = new byte[size]; - this.location = 0; - this.size = size; - } - - private BdfDatabase() { } @Override - public IBdfDatabase getAt(int start, int end) + public IBdfDatabase getCopy(int start, int end) { byte[] database = new byte[end - start]; for(int i=0;i= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(); + + if(!(c == '\n' || c == '\t' || c == ' ')) { + return; + } + + increment(); + } + } + + // In the format "abc\n\t\u0003..." + public String getQuotedString() + { + if(getChar() != '"') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, this); + } + + increment(); + String str = ""; + + while(true) + { + if(offset >= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(); + + // Check for back slashes + if(c == '\\') + { + increment(1); + c = getChar(); + + switch(c) + { + case 'n': + str += "\n"; + break; + case 't': + str += "\t"; + break; + case '"': + str += "\""; + break; + case '\\': + str += "\\"; + break; + case 'u': // \u0000 + { + if(offset + 5 >= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, getPointer(1)); + } + + char[] hex = getCharArray(1, 4); + char unicode = (char)0; + int m = 1; + + for(int j=hex.length-1;j>=0;j--) + { + c = hex[j]; + + if(c >= '0' && c <= '9') { + unicode += (char)(m * (c - '0')); + } + + else if(c >= 'a' && c <= 'f') { + unicode += (char)(m * (c - 'a' + 10)); + } + + else { + throw BdfError.createError(BdfError.ERROR_SYNTAX, getPointer(1 + (hex.length-j-1))); + } + + m *= 16; + } + + str += unicode; + increment(5); + + break; + } + default: + str += "\\" + c; + } + } + + else if(c == '"') { + increment(); + break; + } + + else { + increment(); + str += c; + } + } + + return str; + } + + public boolean isNext(String check) + { + if(check.length() + offset >= data.length) { + return false; + } + + for(int i=0;i= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(i); + c = (char)((c >= 'A' && c <= 'Z') ? (c + 32) : c); + + if(c != check.charAt(i)) { + return false; + } + } + + increment(check.length()); + + return true; + } + + public boolean isInteger() + { + for(int i=offset;i= '0' && c <= '9') { + continue; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, new BdfStringPointer(data, i)); + } + + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, new BdfStringPointer(data, data.length - 1)); + } +} diff --git a/src/bdf/data/IBdfDatabase.java b/src/bdf/data/IBdfDatabase.java index 30cd911..8ca3658 100644 --- a/src/bdf/data/IBdfDatabase.java +++ b/src/bdf/data/IBdfDatabase.java @@ -5,7 +5,7 @@ import java.io.OutputStream; public interface IBdfDatabase { - public IBdfDatabase getAt(int start, int end); + public IBdfDatabase getCopy(int start, int end); public IBdfDatabase getPointer(int location, int size); public IBdfDatabase getPointer(int location); @@ -17,9 +17,18 @@ public interface IBdfDatabase public byte[] getBytes(); public byte[] getBytes(int start, int size); + public byte getByte(); public byte getByte(int i); public String getString(); - public void setBytes(int pos, byte[] bytes); + public void setBytes(byte[] bytes, int offset, int length); + public void setBytes(byte[] bytes, int offset); + public void setBytes(byte[] bytes); + + public void setBytes(IBdfDatabase bytes, int offset, int length); + public void setBytes(IBdfDatabase bytes, int offset); + public void setBytes(IBdfDatabase bytes); + public void setByte(int pos, byte b); + public void setByte(byte b); } diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index aa9b645..35d53f4 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -5,21 +5,68 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Iterator; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; +import bdf.util.BdfError; import bdf.util.DataHelpers; public class BdfArray implements IBdfType, Iterable { - protected ArrayList elements = new ArrayList(); + protected ArrayList elements; protected BdfLookupTable lookupTable; - BdfArray(BdfLookupTable lookupTable) { + BdfArray(BdfLookupTable lookupTable, BdfStringPointer ptr) + { this.lookupTable = lookupTable; + this.elements = new ArrayList(); + + ptr.increment(); + + // [..., ...] + while(true) + { + ptr.ignoreBlanks(); + + if(ptr.getChar() == ']') { + ptr.increment(); + return; + } + + add(new BdfObject(lookupTable, ptr)); + + // There should be a comma after this + ptr.ignoreBlanks(); + + char c = ptr.getChar(); + + if(c == ']') { + ptr.increment(); + return; + } + + if(c != ',') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + } + } + + BdfArray(BdfLookupTable lookupTable, int size) + { + this.lookupTable = lookupTable; + this.elements = new ArrayList(size); + + for(int i=0;i(); // Create an iterator value to loop over the data int i = 0; @@ -54,12 +101,12 @@ public class BdfArray implements IBdfType, Iterable } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) { int pos = 0; for(BdfObject o : elements) { - pos += o.serialize(database.getPointer(pos), locations); + pos += o.serialize(database.getPointer(pos), locations, map, (byte)0); } return pos; diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index ed60d51..add1dba 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import bdf.data.IBdfDatabase; import bdf.util.DataHelpers; @@ -26,8 +28,8 @@ class BdfLookupTable implements IBdfType for(int i=0;i uses[loc_0]) + { + int v_l = locations[loc_0]; + locations[loc_0] = locations[loc_1]; + locations[loc_1] = v_l; + + int v_u = uses[loc_0]; + uses[loc_0] = uses[loc_1]; + uses[loc_1] = v_u; + + int v_m = map_copy[j]; + map_copy[j] = map_copy[j+1]; + map_copy[j+1] = v_m; + + changed = true; + } + } + + if(!changed) { + return map_copy; + } + } + + return map_copy; + } + + public int[] serializeGetLocations(int[] locations) + { + int[] uses = new int[keys.size()]; int next = 0; - reader.bdf.getLocationUses(locations); + reader.bdf.getLocationUses(uses); - for(int i=0;i 0) { + for(int i=0;i 0) { locations[i] = next; next += 1; } else { @@ -111,7 +154,18 @@ class BdfLookupTable implements IBdfType } } - return locations; + int[] map = new int[next]; + next = 0; + + for(int i=0;i elements = new ArrayList(); protected BdfLookupTable lookupTable; + BdfNamedList(BdfLookupTable lookupTable, BdfStringPointer ptr) + { + this.lookupTable = lookupTable; + ptr.increment(); + + // {"key": ..., "key2": ...} + while(true) + { + ptr.ignoreBlanks(); + + char c = ptr.getChar(); + + if(c == '}') { + ptr.increment(); + break; + } + + if(c != '"') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + String key = ptr.getQuotedString(); + + // There should be a colon after this + ptr.ignoreBlanks(); + if(ptr.getChar() != ':') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + + set(key, new BdfObject(lookupTable, ptr)); + + // There should be a comma after this + ptr.ignoreBlanks(); + + c = ptr.getChar(); + + if(c == '}') { + ptr.increment(); + return; + } + + if(c != ',') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + } + } + BdfNamedList(BdfLookupTable lookupTable) { this.lookupTable = lookupTable; } @@ -33,13 +89,45 @@ public class BdfNamedList implements IBdfType // Loop over the data while(i < data.size()) { - // Get the key - int key = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); - i += 4; - // Get the object - int object_size = BdfObject.getSize(data.getPointer(i)); + int key_size = 0; + IBdfDatabase flag_ptr = data.getPointer(i); + int object_size = BdfObject.getSize(flag_ptr); + byte key_size_bytes = BdfObject.getParentFlags(flag_ptr); BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size)); + i += object_size; + + switch(key_size_bytes) + { + case 2: + key_size = 1; + break; + case 1: + key_size = 2; + break; + case 0: + key_size = 4; + break; + } + + // Get the key + ByteBuffer key_buff = DataHelpers.getByteBuffer(data.getPointer(i, key_size)); + int key = 0; + + switch(key_size_bytes) + { + case 2: + key = 0xff & key_buff.get(); + break; + case 1: + key = 0xffff & key_buff.getShort(); + break; + case 0: + key = key_buff.getInt(); + break; + } + + i += key_size; // Create a new element and save some data to it Element element = new Element(); @@ -48,24 +136,41 @@ public class BdfNamedList implements IBdfType // Add the object to the elements list elements.add(element); - - // Increase the iterator by the amount of bytes - i += object_size; } } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) { int pos = 0; for(Element o : elements) { - database.setBytes(pos, DataHelpers.serializeInt(locations[o.key])); + int location = locations[o.key]; + byte size_bytes_tag; + byte size_bytes; - int size = o.object.serialize(database.getPointer(pos + 4), locations); + if(location > 65535) { // >= 2 ^ 16 + size_bytes_tag = 0; + size_bytes = 4; + } else if(location > 255) { // >= 2 ^ 8 + size_bytes_tag = 1; + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes_tag = 2; + size_bytes = 1; + } - pos += size + 4; + int size = o.object.serialize(database.getPointer(pos), locations, map, size_bytes_tag); + int offset = pos + size; + + byte[] bytes = DataHelpers.serializeInt(location); + + for(int i=0;i 65535) { // >= 2 ^ 16 + size += 4; + } else if(location > 255) { // >= 2 ^ 8 + size += 2; + } else { // < 2 ^ 8 + size += 1; + } + size += o.object.serializeSeeker(locations); } @@ -231,14 +345,6 @@ public class BdfNamedList implements IBdfType return keys; } - public int getKeyLocation(String key) { - return lookupTable.getLocation(key.getBytes()); - } - - public String getKeyName(int key) { - return new String(lookupTable.getName(key)); - } - public boolean contains(String key) { return contains(lookupTable.getLocation(key.getBytes())); } diff --git a/src/bdf/types/BdfObject.java b/src/bdf/types/BdfObject.java index 0f6fc1e..473682b 100644 --- a/src/bdf/types/BdfObject.java +++ b/src/bdf/types/BdfObject.java @@ -6,8 +6,11 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import bdf.data.BdfDatabase; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; +import bdf.util.BdfError; import bdf.util.DataHelpers; +import tests.Tests; public class BdfObject implements IBdfType { @@ -15,6 +18,7 @@ public class BdfObject implements IBdfType protected Object object = null; protected byte type = BdfTypes.UNDEFINED; protected BdfLookupTable lookupTable; + protected int last_seek; BdfObject(BdfLookupTable lookupTable) { this.lookupTable = lookupTable; @@ -26,14 +30,16 @@ public class BdfObject implements IBdfType this.lookupTable = lookupTable; // Get the type and database values - byte flags = data.getByte(0); - - type = flags & 0x0f; + int flags = 0xff & data.getByte(0); + type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + int size_bytes = getSizeBytes(flags % 3); + database = data.getPointer(1); // Skip the size bytes if size is stored if(shouldStoreSize(type)) { - database = database.getPointer(4 - (flags & 0b00110000) >> 4); + database = database.getPointer(size_bytes); } // Set the object variable if there is an object specified @@ -46,26 +52,431 @@ public class BdfObject implements IBdfType } } + BdfObject(BdfLookupTable lookupTable, BdfStringPointer ptr) + { + this.lookupTable = lookupTable; + + char c = ptr.getChar(); + + if(c == '{') { + setNamedList(new BdfNamedList(lookupTable, ptr)); + return; + } + + if(c == '[') { + setArray(new BdfArray(lookupTable, ptr)); + return; + } + + if(c == '"') { + setString(ptr.getQuotedString()); + return; + } + + boolean isPrimitiveArray = false; + byte type = 0; + + if(ptr.isNext("int")) { + type = BdfTypes.ARRAY_INTEGER; + isPrimitiveArray = true; + } + + else if(ptr.isNext("long")) { + type = BdfTypes.ARRAY_LONG; + isPrimitiveArray = true; + } + + else if(ptr.isNext("byte")) { + type = BdfTypes.ARRAY_BYTE; + isPrimitiveArray = true; + } + + else if(ptr.isNext("short")) { + type = BdfTypes.ARRAY_SHORT; + isPrimitiveArray = true; + } + + else if(ptr.isNext("bool")) { + type = BdfTypes.ARRAY_BOOLEAN; + isPrimitiveArray = true; + } + + else if(ptr.isNext("double")) { + type = BdfTypes.ARRAY_DOUBLE; + isPrimitiveArray = true; + } + + else if(ptr.isNext("float")) { + type = BdfTypes.ARRAY_FLOAT; + isPrimitiveArray = true; + } + + // Deserialize a primitive array + if(isPrimitiveArray) + { + ptr.ignoreBlanks(); + + if(ptr.getChar() != '(') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + + // Get the size of the array + int size = 0; + + // Get a copy of the pointer + BdfStringPointer ptr2 = ptr.getPointer(0); + + for(;;) + { + if(ptr2.isNext("true") || ptr2.isNext("false")) { + size += 1; + } + + else + { + for(;;) + { + c = ptr2.getChar(); + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + ptr2.increment(); + continue; + } + + if(c == 'B' || c == 'S' || c == 'I' || c == 'L' || c == 'D' || c == 'F') { + ptr2.increment(); + size += 1; + break; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr2); + } + } + + ptr2.ignoreBlanks(); + + if(ptr2.getChar() == ',') { + ptr2.increment(); + ptr2.ignoreBlanks(); + } + + if(ptr2.getChar() == ')') { + ptr2.increment(); + break; + } + } + + Object array = null; + + switch(type) + { + case BdfTypes.ARRAY_BOOLEAN: + array = new boolean[size]; + break; + case BdfTypes.ARRAY_BYTE: + array = new byte[size]; + break; + case BdfTypes.ARRAY_DOUBLE: + array = new double[size]; + break; + case BdfTypes.ARRAY_FLOAT: + array = new float[size]; + break; + case BdfTypes.ARRAY_INTEGER: + array = new int[size]; + break; + case BdfTypes.ARRAY_LONG: + array = new long[size]; + break; + case BdfTypes.ARRAY_SHORT: + array = new short[size]; + break; + } + + for(int i=0;;i++) + { + if(ptr.isNext("true")) + { + if(type != BdfTypes.ARRAY_BOOLEAN) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + boolean[] a = (boolean[]) array; + a[i] = true; + } + + else if(ptr.isNext("false")) + { + if(type != BdfTypes.ARRAY_BOOLEAN) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + boolean[] a = (boolean[]) array; + a[i] = false; + } + + else + { + // Parse a number + String number = ""; + + for(;;) + { + c = ptr.getChar(); + + if(ptr.getDataLocation() > ptr.getDataLength()) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); + } + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + ptr.increment(); + number += c; + continue; + } + + switch(c) + { + case 'D': + { + if(type != BdfTypes.ARRAY_DOUBLE) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + double[] a = (double[]) array; + a[i] = Double.parseDouble(number); + + ptr.increment(); + break; + } + + case 'F': + { + if(type != BdfTypes.ARRAY_FLOAT) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + float[] a = (float[]) array; + a[i] = Float.parseFloat(number); + + ptr.increment(); + break; + } + + case 'I': + { + if(type != BdfTypes.ARRAY_INTEGER) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + int[] a = (int[]) array; + a[i] = Integer.parseInt(number); + + ptr.increment(); + break; + } + + case 'L': + { + if(type != BdfTypes.ARRAY_LONG) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + long[] a = (long[]) array; + a[i] = Long.parseLong(number); + + ptr.increment(); + break; + } + + case 'S': + { + if(type != BdfTypes.ARRAY_SHORT) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + short[] a = (short[]) array; + a[i] = Short.parseShort(number); + + ptr.increment(); + break; + } + + case 'B': + { + if(type != BdfTypes.ARRAY_BYTE) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + byte[] a = (byte[]) array; + a[i] = Byte.parseByte(number); + + ptr.increment(); + break; + } + + default: + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + break; + } + } + + // int (420I, 23I ) + + ptr.ignoreBlanks(); + + if(ptr.getChar() == ',') { + ptr.increment(); + ptr.ignoreBlanks(); + } + + if(ptr.getChar() == ')') { + ptr.increment(); + break; + } + } + + switch(type) + { + case BdfTypes.ARRAY_BOOLEAN: + setBooleanArray((boolean[])array); + break; + case BdfTypes.ARRAY_BYTE: + setByteArray((byte[])array); + break; + case BdfTypes.ARRAY_DOUBLE: + setDoubleArray((double[])array); + break; + case BdfTypes.ARRAY_FLOAT: + setFloatArray((float[])array); + break; + case BdfTypes.ARRAY_INTEGER: + setIntegerArray((int[])array); + break; + case BdfTypes.ARRAY_LONG: + setLongArray((long[])array); + break; + case BdfTypes.ARRAY_SHORT: + setShortArray((short[])array); + break; + } + + return; + } + + if(ptr.isNext("true")) { + setBoolean(true); + return; + } + + if(ptr.isNext("false")) { + setBoolean(false); + return; + } + + if(ptr.isNext("undefined")) { + return; + } + + // Parse a number + String number = ""; + + for(;;) + { + c = ptr.getChar(); + ptr.increment(); + + if(ptr.getDataLocation() > ptr.getDataLength()) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); + } + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + number += c; + continue; + } + + switch(c) + { + case 'D': + setDouble(Double.parseDouble(number)); + return; + case 'F': + setFloat(Float.parseFloat(number)); + return; + case 'B': + setByte(Byte.parseByte(number)); + return; + case 'S': + setShort(Short.parseShort(number)); + return; + case 'I': + setInteger(Integer.parseInt(number)); + return; + case 'L': + setLong(Long.parseLong(number)); + return; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + } + private boolean shouldStoreSize(byte b) { return b > 7; } + static private int getSizeBytes(int size_bytes_tag) + { + switch(size_bytes_tag) + { + case 0: return 4; + case 1: return 2; + case 2: return 1; + default: return 4; + } + } + + static byte getParentFlags(IBdfDatabase db) + { + int flags = 0xff & db.getByte(0); + + byte type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + + byte size_bytes = (byte)(flags % 3); + flags = (byte)((flags - size_bytes) / 3); + + byte parent_flags = (byte)(flags % 3); + flags = (byte)((flags - parent_flags) / 3); + + return parent_flags; + } + static int getSize(IBdfDatabase db) { - byte type = db.getByte(0); + int flags = 0xff & db.getByte(0); + byte type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + + int size_bytes = getSizeBytes(flags % 3); int size = getSize(type); if(size != -1) { return size; } - return DataHelpers.getByteBuffer(db.getPointer(1, 4)).getInt() + 5; + ByteBuffer size_buff = DataHelpers.getByteBuffer(db.getPointer(1, size_bytes)); + + switch(size_bytes) + { + case 4: return size_buff.getInt(); + case 2: return (0xffff & size_buff.getShort()); + case 1: return (0xff & size_buff.get()); + } + + return 0; } static int getSize(byte type) { - type &= 0x0f; - switch(type) { case BdfTypes.BOOLEAN: @@ -98,40 +509,63 @@ public class BdfObject implements IBdfType } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte parent_flags) { - int size; - + int size = last_seek; boolean storeSize = shouldStoreSize(type); + byte size_bytes_tag = 0; + int size_bytes = 0; + + if(storeSize) + { + if(size > 65535) { // >= 2 ^ 16 + size_bytes_tag = 0; + size_bytes = 4; + } else if(size > 255) { // >= 2 ^ 8 + size_bytes_tag = 1; + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes_tag = 2; + size_bytes = 1; + } + } + + int offset = size_bytes + 1; + byte flags = (byte)(type + (size_bytes_tag * 18) + (parent_flags * 3 * 18)); + // Objects switch(type) { case BdfTypes.ARRAY: - size = ((BdfArray)object).serialize(database.getPointer(5), locations) + 5; + size = ((BdfArray)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; break; case BdfTypes.NAMED_LIST: - size = ((BdfNamedList)object).serialize(database.getPointer(5), locations) + 5; + size = ((BdfNamedList)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; break; case BdfTypes.STRING: byte[] str = ((String)object).getBytes(); - size = str.length + 5; - database.setBytes(5, str); + size = str.length + offset; + database.setBytes(str, offset); break; default: - int o = storeSize ? 5 : 1; - size = this.database.size() + o; - database.setBytes(o, this.database.getBytes()); + size = this.database.size() + offset; + database.setBytes(this.database, offset); break; } - database.setByte(0, type); + database.setByte(0, flags); - if(storeSize) { - database.setBytes(1, DataHelpers.serializeInt(size - 5)); + if(storeSize) + { + byte[] bytes = DataHelpers.serializeInt(size); + + for(int i=0;i 65531) { // >= 2 ^ 16 + size_bytes = 4; + } else if(size > 253) { // >= 2 ^ 8 + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes = 1; + } + + size += size_bytes; + last_seek = size; + + return size; } private String calcIndent(BdfIndent indent, int it) { @@ -213,7 +669,7 @@ public class BdfObject implements IBdfType break; case BdfTypes.ARRAY_INTEGER: { - stream.write(("(" + calcIndent(indent, it)).getBytes()); + stream.write(("int(" + calcIndent(indent, it)).getBytes()); int[] array = this.getIntegerArray(); for(int i=0;i 2147483648L || number <= -2147483648L) { + setLong(number); + } else if(number > 32768 || number <= -32768) { + setInteger((int)number); + } else if(number > 128 || number <= -128) { + setShort((short)number); + } else { + setByte((byte)number); + } + + return this; + } + + public long getAutoInt() + { + switch(type) + { + case BdfTypes.BYTE: + return getByte(); + case BdfTypes.SHORT: + return getShort(); + case BdfTypes.INTEGER: + return getInteger(); + case BdfTypes.LONG: + return getLong(); + default: + return 0; + } } // Primitives @@ -601,7 +1085,19 @@ public class BdfObject implements IBdfType } public BdfArray newArray() { - return new BdfArray(lookupTable); + return new BdfArray(lookupTable, 0); + } + + public BdfArray newArray(int size) { + return new BdfArray(lookupTable, size); + } + + public int getKeyLocation(String key) { + return lookupTable.getLocation(key.getBytes()); + } + + public String getKeyName(int key) { + return new String(lookupTable.getName(key)); } public BdfObject setBooleanArray(boolean[] value) { diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java index 606f8ee..04f1666 100644 --- a/src/bdf/types/BdfReader.java +++ b/src/bdf/types/BdfReader.java @@ -2,9 +2,12 @@ package bdf.types; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import bdf.data.BdfDatabase; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; import bdf.util.DataHelpers; @@ -28,34 +31,100 @@ public class BdfReader public BdfReader(IBdfDatabase database) { - if(database.size() < 4) { + if(database.size() == 0) { initNew(); return; } - // Get the lookup table - int lookupTable_size = DataHelpers.getByteBuffer(database.getPointer(0, 4)).getInt(); - lookupTable = new BdfLookupTable(this, database.getPointer(4, lookupTable_size)); + int upto = 0; + + IBdfDatabase flag_ptr = database.getPointer(upto); + byte lookupTable_size_tag = BdfObject.getParentFlags(flag_ptr); + byte lookupTable_size_bytes = 0; + + switch(lookupTable_size_tag) + { + case 0: + lookupTable_size_bytes = 4; + break; + case 1: + lookupTable_size_bytes = 2; + break; + case 2: + lookupTable_size_bytes = 1; + break; + } // Get the rest of the data - int upto = lookupTable_size + 4; - int bdf_size = BdfObject.getSize(database.getPointer(upto)); - bdf = new BdfObject(lookupTable, database.getPointer(upto, bdf_size)); + int bdf_size = BdfObject.getSize(flag_ptr); + IBdfDatabase database_bdf = database.getPointer(upto, bdf_size); + upto += bdf_size; + + // Get the lookup table + ByteBuffer lookupTable_size_buff = DataHelpers.getByteBuffer(database.getPointer(upto, lookupTable_size_bytes)); + int lookupTable_size = 0; + + switch(lookupTable_size_tag) + { + case 0: + lookupTable_size = lookupTable_size_buff.getInt(); + break; + case 1: + lookupTable_size = 0xffff & lookupTable_size_buff.getShort(); + break; + case 2: + lookupTable_size = 0xff & lookupTable_size_buff.get(); + break; + } + + lookupTable = new BdfLookupTable(this, database.getPointer(lookupTable_size_bytes + upto, lookupTable_size)); + bdf = new BdfObject(lookupTable, database_bdf); + } + + public static BdfReader readHumanReadable(String data) + { + BdfReader reader = new BdfReader(); + reader.bdf = new BdfObject(reader.lookupTable, new BdfStringPointer(data.toCharArray(), 0)); + + return reader; } public BdfDatabase serialize() { - int[] locations = lookupTable.serializeGetLocations(); + int[] locations = new int[lookupTable.size()]; + int[] map = lookupTable.serializeGetLocations(locations); int bdf_size = bdf.serializeSeeker(locations); int lookupTable_size = lookupTable.serializeSeeker(locations); - int database_size = bdf_size + lookupTable_size + 4; + + int lookupTable_size_bytes = 0; + byte lookupTable_size_tag = 0; + + if(lookupTable_size > 65535) { // >= 2 ^ 16 + lookupTable_size_tag = 0; + lookupTable_size_bytes = 4; + } else if(lookupTable_size > 255) { // >= 2 ^ 8 + lookupTable_size_tag = 1; + lookupTable_size_bytes = 2; + } else { // < 2 ^ 8 + lookupTable_size_tag = 2; + lookupTable_size_bytes = 1; + } + + int upto = 0; + int database_size = bdf_size + lookupTable_size + lookupTable_size_bytes; BdfDatabase database = new BdfDatabase(database_size); - database.setBytes(0, DataHelpers.serializeInt(lookupTable_size)); + bdf.serialize(database.getPointer(upto, bdf_size), locations, map, lookupTable_size_tag); + upto += bdf_size; - lookupTable.serialize(database.getPointer(4, lookupTable_size), locations); - bdf.serialize(database.getPointer(4 + lookupTable_size, database_size), locations); + byte[] bytes = DataHelpers.serializeInt(lookupTable_size); + + for(int i=0;i 0x7e && c < 0xa1) || c == 0xad) + { + // Will be in the format \u0000 + size += 6; + } + + else + { + size += 1; + } + } + + char[] chars = new char[size]; + int upto = 0; + + for(int i=0;i 0x7e && c < 0xa1) || c == 0xad) + { + // Will be in the format \u0000 + chars[upto] = '\\'; + chars[upto+1] = 'u'; + chars[upto+2] = HEX[(c & 0xf000) >> 12]; + chars[upto+3] = HEX[(c & 0x0f00) >> 8]; + chars[upto+4] = HEX[(c & 0x00f0) >> 4]; + chars[upto+5] = HEX[(c & 0x000f)]; + upto += 6; + } + + else + { + chars[upto] = string_chars[i]; + upto += 1; + } + } + + string_chars = chars; + } // Add quotes to the string - serialized = "\"" + serialized + "\""; + string = "\"" + new String(string_chars) + "\""; // Return the serialized string - return serialized; + return string; } } diff --git a/src/tests/Tests.java b/src/tests/Tests.java index 751e938..be7058f 100755 --- a/src/tests/Tests.java +++ b/src/tests/Tests.java @@ -1,19 +1,15 @@ package tests; -import java.io.IOException; - import bdf.data.IBdfDatabase; -import bdf.types.BdfIndent; -import bdf.types.BdfNamedList; -import bdf.types.BdfObject; -import bdf.types.BdfReader; public class Tests { - static void displayHex(IBdfDatabase db) + public static void displayHex(IBdfDatabase db) { char[] hex_chars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + System.out.print("Size: " + db.size() + ", Hex: "); + for(int i=0;i Date: Tue, 18 Aug 2020 22:52:08 +1000 Subject: [PATCH 6/8] Got something working --- src/bdf/types/BdfArray.java | 6 +-- src/bdf/types/BdfLookupTable.java | 81 +++++++------------------------ src/bdf/types/BdfNamedList.java | 5 +- src/bdf/types/BdfObject.java | 10 ++-- src/bdf/types/BdfReader.java | 17 ++----- src/bdf/types/IBdfType.java | 2 +- src/tests/Tests.java | 80 +++++++++++++++++++++++++++++- 7 files changed, 113 insertions(+), 88 deletions(-) diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index 35d53f4..72e27a8 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -59,7 +59,7 @@ public class BdfArray implements IBdfType, Iterable this.elements = new ArrayList(size); for(int i=0;i } @Override - public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) + public int serialize(IBdfDatabase database, int[] locations, byte flags) { int pos = 0; for(BdfObject o : elements) { - pos += o.serialize(database.getPointer(pos), locations, map, (byte)0); + pos += o.serialize(database.getPointer(pos), locations, (byte)0); } return pos; diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index add1dba..9ef285f 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -55,13 +55,19 @@ class BdfLookupTable implements IBdfType } @Override - public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) + public int serialize(IBdfDatabase database, int[] locations, byte flags) { int upto = 0; - for(int i=0;i uses[loc_0]) - { - int v_l = locations[loc_0]; - locations[loc_0] = locations[loc_1]; - locations[loc_1] = v_l; - - int v_u = uses[loc_0]; - uses[loc_0] = uses[loc_1]; - uses[loc_1] = v_u; - - int v_m = map_copy[j]; - map_copy[j] = map_copy[j+1]; - map_copy[j+1] = v_m; - - changed = true; - } - } - - if(!changed) { - return map_copy; - } - } - - return map_copy; - } - - public int[] serializeGetLocations(int[] locations) + public int[] serializeGetLocations() { + int[] locations = new int[keys.size()]; int[] uses = new int[keys.size()]; int next = 0; @@ -153,19 +117,8 @@ class BdfLookupTable implements IBdfType locations[i] = -1; } } - - int[] map = new int[next]; - next = 0; - - for(int i=0;i 7; } @@ -509,7 +513,7 @@ public class BdfObject implements IBdfType } @Override - public int serialize(IBdfDatabase database, int[] locations, int[] map, byte parent_flags) + public int serialize(IBdfDatabase database, int[] locations, byte parent_flags) { int size = last_seek; boolean storeSize = shouldStoreSize(type); @@ -538,11 +542,11 @@ public class BdfObject implements IBdfType switch(type) { case BdfTypes.ARRAY: - size = ((BdfArray)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; + size = ((BdfArray)object).serialize(database.getPointer(offset), locations, (byte)0) + offset; break; case BdfTypes.NAMED_LIST: - size = ((BdfNamedList)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; + size = ((BdfNamedList)object).serialize(database.getPointer(offset), locations, (byte)0) + offset; break; case BdfTypes.STRING: diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java index 04f1666..8f90eff 100644 --- a/src/bdf/types/BdfReader.java +++ b/src/bdf/types/BdfReader.java @@ -91,8 +91,7 @@ public class BdfReader public BdfDatabase serialize() { - int[] locations = new int[lookupTable.size()]; - int[] map = lookupTable.serializeGetLocations(locations); + int[] locations = lookupTable.serializeGetLocations(); int bdf_size = bdf.serializeSeeker(locations); int lookupTable_size = lookupTable.serializeSeeker(locations); @@ -115,7 +114,7 @@ public class BdfReader int database_size = bdf_size + lookupTable_size + lookupTable_size_bytes; BdfDatabase database = new BdfDatabase(database_size); - bdf.serialize(database.getPointer(upto, bdf_size), locations, map, lookupTable_size_tag); + bdf.serialize(database.getPointer(upto, bdf_size), locations, lookupTable_size_tag); upto += bdf_size; byte[] bytes = DataHelpers.serializeInt(lookupTable_size); @@ -124,7 +123,7 @@ public class BdfReader database.setByte(i + upto, bytes[i - lookupTable_size_bytes + 4]); } - lookupTable.serialize(database.getPointer(upto + lookupTable_size_bytes, lookupTable_size), locations, map, (byte)0); + lookupTable.serialize(database.getPointer(upto + lookupTable_size_bytes, lookupTable_size), locations, (byte)0); return database; } @@ -167,14 +166,4 @@ public class BdfReader public void serializeHumanReadable(OutputStream stream) throws IOException { serializeHumanReadable(stream, new BdfIndent("", "")); } - - public static BdfReader fromHumanReadable(IBdfDatabase data) - { - BdfReader reader = new BdfReader(); - BdfObject bdf = reader.getObject(); - - - - return reader; - } } diff --git a/src/bdf/types/IBdfType.java b/src/bdf/types/IBdfType.java index b69aac6..0861136 100644 --- a/src/bdf/types/IBdfType.java +++ b/src/bdf/types/IBdfType.java @@ -8,7 +8,7 @@ import bdf.data.IBdfDatabase; interface IBdfType { void getLocationUses(int[] locations); - int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags); + int serialize(IBdfDatabase database, int[] locations, byte flags); int serializeSeeker(int[] locations); void serializeHumanReadable(OutputStream stream, BdfIndent indent, int it) throws IOException; diff --git a/src/tests/Tests.java b/src/tests/Tests.java index be7058f..0806117 100755 --- a/src/tests/Tests.java +++ b/src/tests/Tests.java @@ -1,9 +1,57 @@ package tests; +import java.io.FileOutputStream; +import java.io.IOException; + +import bdf.classes.IBdfClassManager; import bdf.data.IBdfDatabase; +import bdf.file.BdfFileManager; +import bdf.types.BdfArray; +import bdf.types.BdfIndent; +import bdf.types.BdfNamedList; +import bdf.types.BdfObject; +import bdf.types.BdfReader; +import bdf.types.BdfTypes; +import bdf.util.FileHelpers; public class Tests { + private static class Storage implements IBdfClassManager + { + String name; + int type; + int age; + + public Storage(String name, int age, int type) { + this.name = name; + this.age = age; + this.type = type; + } + + @Override + public void BdfClassLoad(BdfObject bdf) + { + + } + + @Override + public void BdfClassSave(BdfObject bdf) + { + BdfNamedList nl = bdf.getNamedList(); + + nl.set("name", bdf.newObject().setString(name)); + + if(age != -1) { + nl.set("age", bdf.newObject().setAutoInt(age)); + } + + if(type != -1) { + nl.set("type", bdf.newObject().setAutoInt(type)); + } + } + + } + public static void displayHex(IBdfDatabase db) { char[] hex_chars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; @@ -23,9 +71,39 @@ public class Tests System.out.println(); } - public static void main(String[] args) + public static void main(String[] args) throws IOException { + BdfReader reader = new BdfReader(); + BdfObject bdf = reader.getObject(); + bdf.getKeyLocation("type"); + bdf.getKeyLocation("qwerty"); + bdf.getKeyLocation("age"); + bdf.getKeyLocation("name"); + + Storage[] items = { + new Storage("test1", -1, 1), + new Storage("test2", 69, 2), + new Storage("test3", 420, -1), + new Storage("test4", 23, 3), + new Storage("test5", -1, -1), + }; + + BdfArray array = bdf.newArray(5); + bdf.setArray(array); + + for(int i=0;i Date: Wed, 19 Aug 2020 13:40:15 +1000 Subject: [PATCH 7/8] Updated gitignore --- .gitignore | 1 + data.hbdf | 22 ---------------------- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 data.hbdf diff --git a/.gitignore b/.gitignore index 894b6fe..2700323 100755 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /bin/ /*.bdf +/*.hbdf diff --git a/data.hbdf b/data.hbdf deleted file mode 100644 index bbdc4fb..0000000 --- a/data.hbdf +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "Josua", - "age": 17I, - "anArray": [ - "hi =)", - 69B - ], - "anIntArray": int( - 432I, - 234I, - 69I, - 2I, - 423I - ), - "array2": [ - bool( - false, true , - true , true , - true , false, - ) - ] -} From 960d0c3b17ef110ac91bb96fef55f4c4b3e3439a Mon Sep 17 00:00:00 2001 From: jsrobson10 Date: Wed, 9 Sep 2020 14:02:38 +1000 Subject: [PATCH 8/8] Updated the README, added support for non-real integers, fixed issues when trying to parse garbage data, cleaned up some code. --- README.md | 238 ++++++++++++++++---------- src/bdf/classes/IBdfClassManager.java | 9 - src/bdf/data/BdfStringPointer.java | 89 ++++++++-- src/bdf/file/.gitignore | 1 - src/bdf/file/BdfFileManager.java | 89 ---------- src/bdf/types/BdfArray.java | 9 +- src/bdf/types/BdfLookupTable.java | 12 +- src/bdf/types/BdfNamedList.java | 10 +- src/bdf/types/BdfObject.java | 231 +++++++++++++++++++++---- src/bdf/types/BdfReader.java | 125 ++++++++------ src/bdf/types/BdfReaderHuman.java | 29 ++++ src/bdf/util/BdfError.java | 28 +-- src/bdf/util/FileHelpers.java | 87 ---------- src/tests/Tests.java | 93 +++------- 14 files changed, 568 insertions(+), 482 deletions(-) delete mode 100755 src/bdf/classes/IBdfClassManager.java delete mode 100644 src/bdf/file/.gitignore delete mode 100644 src/bdf/file/BdfFileManager.java create mode 100644 src/bdf/types/BdfReaderHuman.java delete mode 100644 src/bdf/util/FileHelpers.java diff --git a/README.md b/README.md index ad08ed2..fc37d69 100644 --- a/README.md +++ b/README.md @@ -8,23 +8,19 @@ - Creating an object - Arrays - Named lists -- Saving classes -- Implementation details +- Human readable representation +- Special notes ### Overview -Binary Data Format (or BDF) is designed to store data in a tree-like binary structure, -like Notch's NBT format, but also open source and free like JSON. The format is -fast and allows multiple data types. It uses 32-bit integers, so BDF files can -be fast and work well on 32-bit systems, but have a maximum size of 2 GB. -BDF allows human readable serialization to see what is going on for debugging -purposes, but it currently can't parse the human readable serialized string to an object. - +Binary Data Format (BDF) is a statically typed data representation +format. It was made to be free, fast, compact, and seamlessly +convertable between its human readable and binary representations. ### Languages -- Java - C++ +- Java ### Data types @@ -70,8 +66,15 @@ int v = bdf.getInteger(); // Set an integer bdf.setInteger(5); -// Set a "smart" integer -bdf.setSmartInteger(53); +// Set an integer with an automatic type +bdf.setAutoInt(53); + +// Set a primitive array of ints +int intArray[] = {3, 4, 5, 6}; +bdf.setIntegerArray(intArray); + +// Get a byte array +byte[] byteArray = bdf.getByteArray(); // Get the type of variable of the object int type = bdf.getType(); @@ -83,38 +86,27 @@ if(type == BdfTypes.INTEGER) } // Serialize the BDF object -IBdfDatabase data = bdf.serialize(); +byte[] data = bdf->serialize(&data, &data_size); -// Load another BDF object with the serialized bytes -BdfObject bdf2 = new BdfObject(new BdfDatabase(data)); +// Load another reader object from the serialized bytes +BdfReader reader2 = new BdfReader(data); -``` +/* + A reader object can be serialized to the human readable + representation as a string or sent over a stream +*/ +reader2.serializeHumanReadable(System.out, new BdfIndent("\t", "\n")); +String data_hr = reader2.serializeHumanReadable(new BdfIndent("\t", "\n")); -A file manager instance can be used in the same way as a reader object, -but it also needs a String parameter for the path of the file. The file -manager instance also has the capacity to use compression (by default this -uses the GZIP compression algorithm). - -```java - -// Open a file with compression enabled -BdfFileManager reader = new BdfFileManager("file.bdf", true); - -// Save the database -reader.saveDatabase(); - -// The file can be casted to a BdfReader -BdfReader reader2 = (BdfReader) reader; - -// Can be used just as any reader instance -BdfObject bdf = reader.getObject(); +// A reader object can be loaded from a human readable object +BdfReader reader3 = new BdfReaderHuman(data_hr); ``` ### Arrays -Arrays can be used to store lists of information, they hold instances of -BdfObject. Arrays have support for Iterators and are an instance of Iterable. +Arrays can be used to store chunks of information, they hold instances of +BdfObject. Arrays can also be iterated over just like any other array. ```java @@ -134,17 +126,11 @@ array.remove(3); array.set(4, bdf.newObject().setString("A String")); // Add an object to an array -array.add(bdf.newObject().setByte(53)); +array.add(bdf.newObject().setByte((byte)53)); // Set the array to the bdf object bdf.setArray(array); -// Iterate over an array -for(BdfObject o : array) -{ - -} - ``` ### Named lists @@ -156,26 +142,26 @@ can be created similar to an array. ```java BdfReader reader = new BdfReader(); -BdfObject bdf = new BdfObject(); +BdfObject bdf = reader.getObject(); // New named list -BdfNamedList nl = bdf.newNamedList(); +BdfNamedList list = bdf.newNamedList(); // Set an element to the named list -nl.set("key1", bdf.newObject().setInteger(5)); +list.set("key1", bdf.newObject().setInteger(5)); // Use ids instead of strings for optimisation // if set/get is being called multiple times // on the same key. -int key2 = nl.getKeyLocation("key2"); -nl.set(key2, bdf.newObject().setFloat(42.0F)); +int key2 = bdf.getKeyLocation("key2"); +list.set(key2, bdf.newObject().setFloat(42.0F)); // Get an elements value int v = list.get("key1").getInteger(); // Check if an element exists -boolean has_key = list.contains("key1"); +bool has_key = list.contains("key1"); // Get the lists keys int[] keys = list.getKeys(); @@ -184,64 +170,132 @@ int[] keys = list.getKeys(); for(int key : keys) { // Get the keys name - String key_name = nl.getKeyName(key); + String key_name = bdf.getKeyName(key); } ``` -### Further optimisations +### Human readable representation +A big part of binary data format is the human readable +representation. It has a JSON-like syntax. +This can be used with config files and to modify/view +binaries. A big advantage to using the human readable +representation in configuration files is its support +for comments. -### Implementation details +```hbdf -All integer data types are in the Big Endian layout. +/* + A Named List is represented + by an opening tag and a closing + tag { } +*/ +{ + /* + A key value pair can be stored + within a Named List with a string + property + */ + "hello": "world", -**Flags (1 unsigned byte)** -This holds 3 values: -- Type (0-17) -- Size type (0-2) -- Parent payload (0-2) + /* + Integers can be stored here too. + They have a character at the end + to say what type they are. + + The tag at the end can be: + - I: Integer - a value between -2^31 and 2^31 - 1 + - S: Short - a value between -32768 and 32767 + - L: Long - a value between -2^63 and 2^63 - 1 + - B: Byte - a value between -128 and 127 + - D: Double - has 15 decimal digits of precision + - F: Float - has 7 decimal digits of precision + */ + "number": 42I, + "byte": -23B, + "decimal": 73.5D, -**Type** -``` -0: UNDEFINED (0 bytes) + /* + This is a boolean. It can + be true or false. + */ + "boolTest": false, -1: BOOLEAN (1 byte, 0x00 or 0x01) -2: INTEGER (4 bytes) -3: LONG (8 bytes) -4: SHORT (2 bytes) -5: BYTE (1 byte) -6: DOUBLE (8 bytes) -7: FLOAT (4 bytes) + /* + Primitive arrays are represented + by a type, an opening tag, and a + closing tag. They are like an array + but they contain only 1 data type. -8: STRING -9: ARRAY -10: NAMED_LIST + The tag at the start can be: + - int + - short + - long + - byte + - double + - float + - bool + */ + "intArray": int ( + 64I, 42I, 63I, + 22I, 96I, -12I, + ), -11: ARRAY_BOOLEAN -12: ARRAY_INTEGER -13: ARRAY_LONG -14: ARRAY_SHORT -15: ARRAY_BYTE -16: ARRAY_DOUBLE -17: ARRAY_FLOAT + /* + The double and float types support + Infinity, -Infinity, and NaN. + + They also support both really + high and really low value numbers. + */ + "doubleArray": double ( + 42.5D, -20D, 400D, + NaND, -InfinityD, InfinityD, + 5.3e-200F, 4e+500F, 2.2e200F, + ) + + /* + Arrays are enclosed by an opening + tag and a closing tag [ ] + + Like the Named List, it can hold + any data type. + */ + "people": [ + {"name": "foo", "age": 60B}, + {"name": "bar", "age": 21B}, + ], + + // This is a single-line comment + + /* This is a multi-line comment */ +} ``` -**Size Type** -This value holds info for how big the size of -the size of the payload is, in bytes. The purpose -of this is to reduce the size as much as possible -by throwing out unneccicary zeros. +### Special notes -**Object** -- Flags (unsigned byte, 1 byte) -- Size (variable length) -- Payload (Any type, variable length) +Don't mix bdf types between different +readers, this will cause problems. -**NamedList** -- Key ID (variable length) -- Payload (Object, variable length) +```java + +BdfReader reader1 = new BdfReader(); +BdfReader reader2 = new BdfReader(); + +BdfObject bdf1 = reader1.getObject(); +BdfObject bdf2 = reader2.getObject(); + +// Don't do this +bdf1.setNamedList(bdf2.newNamedList()); + +// Or this +bdf1.setArray(bdf2.newArray()); + +// Or this +BdfNamedList nl = bdf1.newArray(); +nl.set("illegal", bdf2.newObject().setString("action")); + +``` -**Array** -- Payload (Object, variable length) diff --git a/src/bdf/classes/IBdfClassManager.java b/src/bdf/classes/IBdfClassManager.java deleted file mode 100755 index 68a0f5d..0000000 --- a/src/bdf/classes/IBdfClassManager.java +++ /dev/null @@ -1,9 +0,0 @@ -package bdf.classes; - -import bdf.types.BdfObject; - -public interface IBdfClassManager -{ - public void BdfClassLoad(BdfObject bdf); - public void BdfClassSave(BdfObject bdf); -} diff --git a/src/bdf/data/BdfStringPointer.java b/src/bdf/data/BdfStringPointer.java index 1efee1e..f62c5d0 100644 --- a/src/bdf/data/BdfStringPointer.java +++ b/src/bdf/data/BdfStringPointer.java @@ -7,6 +7,11 @@ public class BdfStringPointer char[] data; int offset; + public BdfStringPointer(char[] data, int offset) { + this.data = data; + this.offset = offset; + } + public BdfStringPointer getPointer(int offset) { return new BdfStringPointer(data, this.offset + offset); } @@ -45,11 +50,6 @@ public class BdfStringPointer public char[] getCharArray(int length) { return getCharArray(0, length); } - - public BdfStringPointer(char[] data, int offset) { - this.data = data; - this.offset = offset; - } public char getChar(int i) { return data[offset + i]; @@ -61,7 +61,7 @@ public class BdfStringPointer public void ignoreBlanks() { - while(true) + for(;;) { if(offset >= data.length) { throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); @@ -69,7 +69,54 @@ public class BdfStringPointer char c = getChar(); - if(!(c == '\n' || c == '\t' || c == ' ')) { + // Comments + if(c == '/' && offset < data.length) + { + char c2 = getChar(1); + + // Line comment + if(c2 == '/') + { + for(;;) + { + if(offset + 1 >= data.length) { + break; + } + + increment(); + c = getChar(); + + if(c == '\n') { + break; + } + } + } + + // Multi-line comment + else if(c2 == '*') + { + for(;;) + { + if(offset + 1 >= data.length) { + throw BdfError.createError(BdfError.ERROR_UNESCAPED_COMMENT, getPointer(-1)); + } + + increment(); + c = getChar(); + + if(c == '*' && offset < data.length && getChar(1) == '/') { + increment(); + break; + } + } + } + + else { + return; + } + } + + else if(!(c == '\n' || c == '\t' || c == ' ')) { return; } @@ -87,10 +134,10 @@ public class BdfStringPointer increment(); String str = ""; - while(true) + for(;;) { if(offset >= data.length) { - throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + throw BdfError.createError(BdfError.ERROR_UNESCAPED_STRING, this); } char c = getChar(); @@ -98,27 +145,35 @@ public class BdfStringPointer // Check for back slashes if(c == '\\') { - increment(1); + increment(); c = getChar(); switch(c) { case 'n': str += "\n"; + increment(); break; case 't': str += "\t"; + increment(); break; case '"': str += "\""; + increment(); break; case '\\': str += "\\"; + increment(); + break; + case '\n': + str += "\n"; + increment(); break; case 'u': // \u0000 { if(offset + 5 >= data.length) { - throw BdfError.createError(BdfError.ERROR_END_OF_FILE, getPointer(1)); + throw BdfError.createError(BdfError.ERROR_UNESCAPED_STRING, getPointer(1)); } char[] hex = getCharArray(1, 4); @@ -154,6 +209,10 @@ public class BdfStringPointer } } + else if(c == '\n') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, this); + } + else if(c == '"') { increment(); break; @@ -170,16 +229,12 @@ public class BdfStringPointer public boolean isNext(String check) { - if(check.length() + offset >= data.length) { + if(check.length() + offset > data.length) { return false; } for(int i=0;i= data.length) { - throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); - } - char c = getChar(i); c = (char)((c >= 'A' && c <= 'Z') ? (c + 32) : c); @@ -219,6 +274,8 @@ public class BdfStringPointer continue; case '.': continue; + case '-': + continue; } if(c >= '0' && c <= '9') { diff --git a/src/bdf/file/.gitignore b/src/bdf/file/.gitignore deleted file mode 100644 index 097f6c7..0000000 --- a/src/bdf/file/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/BdfFileManager.class diff --git a/src/bdf/file/BdfFileManager.java b/src/bdf/file/BdfFileManager.java deleted file mode 100644 index 0a78daa..0000000 --- a/src/bdf/file/BdfFileManager.java +++ /dev/null @@ -1,89 +0,0 @@ -package bdf.file; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.GZIPOutputStream; - -import bdf.data.BdfDatabase; -import bdf.types.BdfObject; -import bdf.types.BdfReader; -import bdf.util.FileHelpers; - -public class BdfFileManager extends BdfReader -{ - protected String path; - private boolean compressed; - - private static BdfDatabase init(String path, boolean compressed) - { - // Get the file - File file = new File(path); - - // Does the file have read access - if(file.canRead()) - { - if(compressed) - { - // Return the files contents as a database - return new BdfDatabase(FileHelpers.readAllCompressedIgnoreErrors(path)); - } - - else - { - // Return the files contents as a database - return new BdfDatabase(FileHelpers.readAllIgnoreErrors(path)); - } - } - - // Return an empty database if there is no read access - return new BdfDatabase(0); - } - - public BdfFileManager(String path, boolean compressed) { - super(init(path, compressed)); - this.compressed = compressed; - this.path = path; - } - - public BdfFileManager(String path) { - this(path, false); - } - - public void saveDatabase(String path) - { - try - { - // Get the file handler - File file = new File(path); - - // Create the parent directories - file.getAbsoluteFile().getParentFile().mkdirs(); - - // Get the database file for output - OutputStream out = new FileOutputStream(path); - - if(compressed) { - out = new GZIPOutputStream(out); - } - - // Write the database to the file - BdfDatabase db = this.serialize(); - db.writeToStream(out); - - // Close the file output stream - out.close(); - } - - catch(IOException e) { - return; - } - } - - public void saveDatabase() { - this.saveDatabase(this.path); - } -} diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index 72e27a8..ca24133 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -8,16 +8,13 @@ import java.util.Iterator; import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; import bdf.util.BdfError; -import bdf.util.DataHelpers; public class BdfArray implements IBdfType, Iterable { protected ArrayList elements; - protected BdfLookupTable lookupTable; BdfArray(BdfLookupTable lookupTable, BdfStringPointer ptr) { - this.lookupTable = lookupTable; this.elements = new ArrayList(); ptr.increment(); @@ -55,7 +52,6 @@ public class BdfArray implements IBdfType, Iterable BdfArray(BdfLookupTable lookupTable, int size) { - this.lookupTable = lookupTable; this.elements = new ArrayList(size); for(int i=0;i BdfArray(BdfLookupTable lookupTable, IBdfDatabase data) { - this.lookupTable = lookupTable; this.elements = new ArrayList(); // Create an iterator value to loop over the data @@ -77,6 +72,10 @@ public class BdfArray implements IBdfType, Iterable // Get the size of the object int size = BdfObject.getSize(data.getPointer(i)); + if(size <= 0 || i + size > data.size()) { + return; + } + // Get the object BdfObject object = new BdfObject(lookupTable, data.getPointer(i, size)); diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index 9ef285f..d599389 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -1,11 +1,7 @@ package bdf.types; -import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; import bdf.data.IBdfDatabase; import bdf.util.DataHelpers; @@ -31,6 +27,10 @@ class BdfLookupTable implements IBdfType int key_size = 0xff & database.getByte(i); i += 1; + if(i + key_size > database.size()) { + return; + } + keys.add(database.getBytes(i, key_size)); i += key_size; @@ -50,6 +50,10 @@ class BdfLookupTable implements IBdfType return keys.size() - 1; } + boolean hasLocation(int location) { + return location >= 0 && location < keys.size(); + } + byte[] getName(int location) { return keys.get(location); } diff --git a/src/bdf/types/BdfNamedList.java b/src/bdf/types/BdfNamedList.java index 8b07efd..bc6d3a4 100644 --- a/src/bdf/types/BdfNamedList.java +++ b/src/bdf/types/BdfNamedList.java @@ -3,7 +3,6 @@ package bdf.types; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import bdf.data.BdfStringPointer; @@ -94,6 +93,11 @@ public class BdfNamedList implements IBdfType IBdfDatabase flag_ptr = data.getPointer(i); int object_size = BdfObject.getSize(flag_ptr); byte key_size_bytes = BdfObject.getParentFlags(flag_ptr); + + if(object_size <= 0 || i + object_size >= data.size()) { + return; + } + BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size)); i += object_size; @@ -127,6 +131,10 @@ public class BdfNamedList implements IBdfType break; } + if(!lookupTable.hasLocation(key)) { + return; + } + i += key_size; // Create a new element and save some data to it diff --git a/src/bdf/types/BdfObject.java b/src/bdf/types/BdfObject.java index 49d4c8c..57fcd43 100644 --- a/src/bdf/types/BdfObject.java +++ b/src/bdf/types/BdfObject.java @@ -1,6 +1,5 @@ package bdf.types; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -10,7 +9,6 @@ import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; import bdf.util.BdfError; import bdf.util.DataHelpers; -import tests.Tests; public class BdfObject implements IBdfType { @@ -42,10 +40,25 @@ public class BdfObject implements IBdfType database = database.getPointer(size_bytes); } + if(database.size() < 0) { + database = new BdfDatabase(0); + type = BdfTypes.UNDEFINED; + return; + } + // Set the object variable if there is an object specified - if(type == BdfTypes.STRING) object = database.getString(); - if(type == BdfTypes.ARRAY) object = new BdfArray(lookupTable, database); - if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(lookupTable, database); + switch(type) + { + case BdfTypes.STRING: + object = database.getString(); + break; + case BdfTypes.ARRAY: + object = new BdfArray(lookupTable, database); + break; + case BdfTypes.NAMED_LIST: + object = new BdfNamedList(lookupTable, database); + break; + } if(object != null) { database = null; @@ -73,6 +86,7 @@ public class BdfObject implements IBdfType return; } + boolean isDecimalArray = false; boolean isPrimitiveArray = false; byte type = 0; @@ -104,11 +118,13 @@ public class BdfObject implements IBdfType else if(ptr.isNext("double")) { type = BdfTypes.ARRAY_DOUBLE; isPrimitiveArray = true; + isDecimalArray = true; } else if(ptr.isNext("float")) { type = BdfTypes.ARRAY_FLOAT; isPrimitiveArray = true; + isDecimalArray = true; } // Deserialize a primitive array @@ -131,7 +147,17 @@ public class BdfObject implements IBdfType for(;;) { - if(ptr2.isNext("true") || ptr2.isNext("false")) { + if(ptr2.getChar() == ')') { + ptr2.increment(); + break; + } + + if( + ptr2.isNext("true") || ptr2.isNext("false") || + ptr2.isNext("infinityd") || ptr2.isNext("-infinityd") || + ptr2.isNext("infinityf") || ptr2.isNext("-infinityf") || + ptr2.isNext("nand") || ptr2.isNext("nanf")) + { size += 1; } @@ -139,9 +165,17 @@ public class BdfObject implements IBdfType { for(;;) { + if(ptr2.getDataLocation() >= ptr2.getDataLength()) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr2.getPointer(-1)); + } + c = ptr2.getChar(); - if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + if(c >= 'a' && c <= 'z') { + c -= 32; + } + + if(c >= '0' && c <= '9' || ((c == '.' || c == 'E') && isDecimalArray) || c == '+' || c == '-') { ptr2.increment(); continue; } @@ -198,6 +232,11 @@ public class BdfObject implements IBdfType for(int i=0;;i++) { + if(ptr.getChar() == ')') { + ptr.increment(); + break; + } + if(ptr.isNext("true")) { if(type != BdfTypes.ARRAY_BOOLEAN) { @@ -218,6 +257,66 @@ public class BdfObject implements IBdfType a[i] = false; } + else if(ptr.isNext("infinityd")) + { + if(type != BdfTypes.ARRAY_DOUBLE) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + double[] a = (double[]) array; + a[i] = Double.POSITIVE_INFINITY; + } + + else if(ptr.isNext("-infinityd")) + { + if(type != BdfTypes.ARRAY_DOUBLE) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + double[] a = (double[]) array; + a[i] = Double.NEGATIVE_INFINITY; + } + + else if(ptr.isNext("nand")) + { + if(type != BdfTypes.ARRAY_DOUBLE) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + double[] a = (double[]) array; + a[i] = Double.NaN; + } + + else if(ptr.isNext("infinityf")) + { + if(type != BdfTypes.ARRAY_FLOAT) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + float[] a = (float[]) array; + a[i] = Float.POSITIVE_INFINITY; + } + + else if(ptr.isNext("-infinityf")) + { + if(type != BdfTypes.ARRAY_FLOAT) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + float[] a = (float[]) array; + a[i] = Float.NEGATIVE_INFINITY; + } + + else if(ptr.isNext("nanf")) + { + if(type != BdfTypes.ARRAY_FLOAT) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + float[] a = (float[]) array; + a[i] = Float.NaN; + } + else { // Parse a number @@ -227,11 +326,15 @@ public class BdfObject implements IBdfType { c = ptr.getChar(); + if(c >= 'a' && c <= 'z') { + c -= 32; + } + if(ptr.getDataLocation() > ptr.getDataLength()) { throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); } - if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + if(c >= '0' && c <= '9' || ((c == '.' || c == 'E') && isDecimalArray) || c == '+' || c == '-') { ptr.increment(); number += c; continue; @@ -372,50 +475,103 @@ public class BdfObject implements IBdfType return; } + if(ptr.isNext("infinityd")) { + setDouble(Double.POSITIVE_INFINITY); + return; + } + + if(ptr.isNext("-infinityd")) { + setDouble(Double.NEGATIVE_INFINITY); + return; + } + + if(ptr.isNext("nand")) { + setDouble(Double.NaN); + return; + } + + if(ptr.isNext("infinityf")) { + setFloat(Float.POSITIVE_INFINITY); + return; + } + + if(ptr.isNext("-infinityf")) { + setFloat(Float.NEGATIVE_INFINITY); + return; + } + + if(ptr.isNext("nanf")) { + setFloat(Float.NaN); + return; + } + if(ptr.isNext("undefined")) { + database = new BdfDatabase(0); return; } // Parse a number String number = ""; + boolean isDecimal = false; for(;;) { c = ptr.getChar(); ptr.increment(); + if(c >= 'a' && c <= 'z') { + c -= 32; + } + if(ptr.getDataLocation() > ptr.getDataLength()) { throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); } - if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + if(c == '.' || c == 'E') { + isDecimal = true; number += c; continue; } - switch(c) - { - case 'D': - setDouble(Double.parseDouble(number)); - return; - case 'F': - setFloat(Float.parseFloat(number)); - return; - case 'B': - setByte(Byte.parseByte(number)); - return; - case 'S': - setShort(Short.parseShort(number)); - return; - case 'I': - setInteger(Integer.parseInt(number)); - return; - case 'L': - setLong(Long.parseLong(number)); - return; + if(c >= '0' && c <= '9' || c == '+' || c == '-') { + number += c; + continue; } - throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + if(isDecimal && !(c == 'D' || c == 'F')) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr.getPointer(-1)); + } + + try + { + switch(c) + { + case 'D': + setDouble(Double.parseDouble(number)); + return; + case 'F': + setFloat(Float.parseFloat(number)); + return; + case 'B': + setByte(Byte.parseByte(number)); + return; + case 'S': + setShort(Short.parseShort(number)); + return; + case 'I': + setInteger(Integer.parseInt(number)); + return; + case 'L': + setLong(Long.parseLong(number)); + return; + default: + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + } + + catch(NumberFormatException e) { + throw BdfError.createError(BdfError.ERROR_NUMBER, ptr.getPointer(-number.length())); + } } } @@ -673,7 +829,7 @@ public class BdfObject implements IBdfType break; case BdfTypes.ARRAY_INTEGER: { - stream.write(("int(" + calcIndent(indent, it)).getBytes()); + stream.write("int(".getBytes()); int[] array = this.getIntegerArray(); for(int i=0;i database.size()) { + initNew(); + return; + } + + // Get the rest of the data + int bdf_size = BdfObject.getSize(flag_ptr); + + if(bdf_size + lookupTable_size_bytes > database.size()) { + initNew(); + return; + } + + IBdfDatabase database_bdf = database.getPointer(upto, bdf_size); + upto += bdf_size; + + // Get the lookup table + ByteBuffer lookupTable_size_buff = DataHelpers.getByteBuffer(database.getPointer(upto, lookupTable_size_bytes)); + int lookupTable_size = 0; + + switch(lookupTable_size_tag) + { + case 0: + lookupTable_size = lookupTable_size_buff.getInt(); + break; + case 1: + lookupTable_size = 0xffff & lookupTable_size_buff.getShort(); + break; + case 2: + lookupTable_size = 0xff & lookupTable_size_buff.get(); + break; + } + + if(lookupTable_size + lookupTable_size_bytes + bdf_size > database.size()) { + initNew(); + return; + } + + lookupTable = new BdfLookupTable(this, database.getPointer(lookupTable_size_bytes + upto, lookupTable_size)); + bdf = new BdfObject(lookupTable, database_bdf); + } + + catch(IndexOutOfBoundsException e) { initNew(); - return; } - - int upto = 0; - - IBdfDatabase flag_ptr = database.getPointer(upto); - byte lookupTable_size_tag = BdfObject.getParentFlags(flag_ptr); - byte lookupTable_size_bytes = 0; - - switch(lookupTable_size_tag) - { - case 0: - lookupTable_size_bytes = 4; - break; - case 1: - lookupTable_size_bytes = 2; - break; - case 2: - lookupTable_size_bytes = 1; - break; - } - - // Get the rest of the data - int bdf_size = BdfObject.getSize(flag_ptr); - IBdfDatabase database_bdf = database.getPointer(upto, bdf_size); - upto += bdf_size; - - // Get the lookup table - ByteBuffer lookupTable_size_buff = DataHelpers.getByteBuffer(database.getPointer(upto, lookupTable_size_bytes)); - int lookupTable_size = 0; - - switch(lookupTable_size_tag) - { - case 0: - lookupTable_size = lookupTable_size_buff.getInt(); - break; - case 1: - lookupTable_size = 0xffff & lookupTable_size_buff.getShort(); - break; - case 2: - lookupTable_size = 0xff & lookupTable_size_buff.get(); - break; - } - - lookupTable = new BdfLookupTable(this, database.getPointer(lookupTable_size_bytes + upto, lookupTable_size)); - bdf = new BdfObject(lookupTable, database_bdf); - } - - public static BdfReader readHumanReadable(String data) - { - BdfReader reader = new BdfReader(); - reader.bdf = new BdfObject(reader.lookupTable, new BdfStringPointer(data.toCharArray(), 0)); - - return reader; } public BdfDatabase serialize() diff --git a/src/bdf/types/BdfReaderHuman.java b/src/bdf/types/BdfReaderHuman.java new file mode 100644 index 0000000..fe9c73f --- /dev/null +++ b/src/bdf/types/BdfReaderHuman.java @@ -0,0 +1,29 @@ +package bdf.types; + +import bdf.data.BdfStringPointer; +import bdf.util.BdfError; + +public class BdfReaderHuman extends BdfReader +{ + public BdfReaderHuman(String data) + { + BdfStringPointer ptr = new BdfStringPointer(data.toCharArray(), 0); + + ptr.ignoreBlanks(); + bdf = new BdfObject(lookupTable, ptr); + + try { + ptr.ignoreBlanks(); + } + + catch(BdfError e) { + if(e.getType() == BdfError.ERROR_END_OF_FILE) { + return; + } else { + throw e; + } + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } +} diff --git a/src/bdf/util/BdfError.java b/src/bdf/util/BdfError.java index d704313..0007e45 100644 --- a/src/bdf/util/BdfError.java +++ b/src/bdf/util/BdfError.java @@ -8,10 +8,16 @@ public class BdfError extends RuntimeException private static final String[] ERRORS = { "Syntax error", "End of file", + "Unescaped comment", + "Unescaped string", + "Number format error", }; public static final int ERROR_SYNTAX = 0; public static final int ERROR_END_OF_FILE = 1; + public static final int ERROR_UNESCAPED_COMMENT = 2; + public static final int ERROR_UNESCAPED_STRING = 3; + public static final int ERROR_NUMBER = 4; public static BdfError createError(int errorID, BdfStringPointer ptr) { @@ -33,13 +39,7 @@ public class BdfError extends RuntimeException continue; } - if(array[i] == '\t') { - at = at / 4 * 4 + 4; - } - - else { - at += 1; - } + at += 1; } int line_size = 0; @@ -47,16 +47,16 @@ public class BdfError extends RuntimeException for(int i=start_of_line;i 1000) { + System.out.println("" + i + ": " + done); + start += 1000; + done = 0; + i += 1; + } + + done += 1; + + rand.read(buffer); + new BdfReader(buffer); } - - reader.serializeHumanReadable(System.out); - - IBdfDatabase db = reader.serialize(); - displayHex(db); - - reader = new BdfReader(db); - - reader.serializeHumanReadable(System.out); } }